Changeset 28973 for main/trunk/greenstone2

Timestamp:

2014-04-14T22:20:14+12:00 (10 years ago)

Author:

ak19

Message:

All the changes required to get the GTI installed on a local greenstone using the latest gs2 src code from svn, since this has the security updates. 1. ApplyXSLT in build-src needed a lot of code additions since runtime-src's gtiaction.cpp needs to send the xml file from stdin using a pipe command. ApplyXSLT used to read stdin from a piped cmd differently, expecting DocStart and DocEnd embedding tags to mark the start and stop of each stream. This is still used by code in BasePlugout, so ApplyXSLT has been modified to take a minus-c parameter when requested to read from stdin without special embedding tag markers, such as when gtiaction calls it. ApplyXSLT uses an internal StreamGobbler class to read the stdin since it takes a while for xml generated by the gti.pl (which is piped in) to come in to ApplyXSLT. 2. The inner StreamGobbler class needed to be added into the ApplyXSLT jar file, so the Makefile.in has been updated. 3. In runtime-src, added in missing header files and updated the code that generated the spreadsheets on GTI, since it was firstly hardcoded to use paths on /home/nzdl, and the code that generated the spreadsheets when running ApplyXSLT/xalan.jar no longer worked as it had been coded.

Location:

main/trunk/greenstone2

Files:

• build-src/src/java/org/nzdl/gsdl/ApplyXSLT.java (modified) (9 diffs)
• build-src/src/java/org/nzdl/gsdl/Makefile.in (modified) (1 diff)
• runtime-src/src/recpt/cgiwrapper.cpp (modified) (1 diff)
• runtime-src/src/recpt/gtiaction.cpp (modified) (4 diffs)
• runtime-src/src/recpt/gtiaction.h (modified) (1 diff)

main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/ApplyXSLT.java

@@ -160 +160 @@
         
       }
+
+      //if(br != null) {
+      //  br.close();
+      //  br = null;
+      //}
+      
     }catch (Exception e)
       {
@@ -166 +172 @@
                 
     return false;
+  }
+  
+    // reads xml from stdin, but no <?DocStart?><?DocEnd?> markers, and sends output to STDOUT
+  private boolean processPipedFromStdIn()
+  {
+      try{
+      //System.err.println("Received nothing\n");      
+      
+      ReadStreamGobbler readInStreamGobbler = new ReadStreamGobbler(System.in, true);
+      readInStreamGobbler.start();
+      readInStreamGobbler.join();
+      String outputstr = readInStreamGobbler.getOutput();
+
+      // Using join() above, even though we use only one streamGobbler thread, and even
+      // though we're not dealing with the input/output/error streams of a Process object.
+      // But the join() call here didn't break things.
+      // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+      
+      boolean result = false;
+      if (mapping_file !=null && !mapping_file.equals("")){
+          result = translateXMLWithMapping(outputstr, null); // null: no outputfile, send to STDOUT
+      }
+      else{
+          result = translateXML(outputstr, null); // null: no outputfile, send to STDOUT
+      }
+      
+      if (!result){
+          System.err.println("Translation Failed!!");
+          return false;
+      } else {
+          return true;   
+      }
+      }catch (Exception e) {
+      System.err.println("Receiving piped data error!" + e.toString());
+      } 
+      return false;
   }
     
@@ -180 +222 @@
     setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT
 
-    transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file)));
+    if(output_file != null) {
+    transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file)));
+    } else {
+    transformer.transform(new StreamSource(str), new StreamResult(System.out));
+    }
     return true;
   }
@@ -437 +483 @@
     String targetlang="";
 
+    boolean readFromStdInFlag = false;
+
     HashMap paramMap = new HashMap();
     int index = -1; // index of the '=' sign in cmdline argument specifying custom parameters to be passed into the XSLT
@@ -470 +518 @@
       else if(args[i].equals("-l") && i+1 < args.length && !args[i+1].startsWith("-")){
     targetlang = args[++i];     
+      }
+      else if(args[i].equals("-c")){
+      readFromStdInFlag = true;
       }
       else if(args[i].equals("-h")){
@@ -489 +540 @@
          
     ApplyXSLT core = null;
-    
+
     if (xml_file.equals("") && !xsl_file.equals("")){//read from pipe line
       if (mapping_file.equals("")){ 
@@ -499 +550 @@
             
       if (core != null){
-    core.process();
+      if(readFromStdInFlag) { // ApplyXSLT was run with -c: read from pipe but no <?DocStart?><?DocEnd?> markers
+          core.processPipedFromStdIn();
+      }
+    core.process(); //read from pipe line, but expecting <?DocStart?><?DocEnd?> embedding markers
       }
       else{
@@ -533 +587 @@
     System.out.println("Usage: ApplyXSLT -x File -t File [-m File] [-o File] [-s sourcelang] [-l targetlang] [param-name=param-value]");
     System.out.println("\t-x specifies the xml file (Note: optional for piped xml data)");
+    System.out.println("\t-c read xml file piped from stdin but without DocStart/DocEnd markers. Writes to stdout");
     System.out.println("\t-t specifies the xsl file");
     System.out.println("\t-m specifies the mapping file (for MARCXMLPlugout.pm only)");
@@ -541 +596 @@
     System.exit(-1);
   }
+
+
+    // StreamGobblers used in reading/writing to Process' input and outputstreams can be re-used more generally.
+    // Here in ApplyXSTL.java we use it to read from a pipe line (stdin piped into this ApplyXSLT.java)
+    // Code based on http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
+    class ReadStreamGobbler extends Thread
+    {
+    InputStream is = null;
+    StringBuffer outputstr = new StringBuffer();
+    boolean split_newlines = false;
+    
+    
+    public ReadStreamGobbler(InputStream is)
+    {
+        this.is = is;
+        split_newlines = false;
+    }
+
+    public ReadStreamGobbler(InputStream is, boolean split_newlines)
+    {
+        this.is = is;
+        this.split_newlines = split_newlines;
+    }
+    
+    public void run()
+    {
+        BufferedReader br = null;
+        try {
+        br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+        String line=null;
+        while ( (line = br.readLine()) != null) {
+            //System.out.println("@@@ GOT LINE: " + line);
+            outputstr.append(line);
+            if(split_newlines) {
+            outputstr.append("\n");
+            }
+        }
+        } catch (IOException ioe) {
+        ioe.printStackTrace();  
+        } finally {
+        closeResource(br);
+        }
+    }
+    
+    public String getOutput() { 
+        return outputstr.toString(); 
+    }
+
+    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html
+    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks
+    public void closeResource(Closeable resourceHandle) {
+        try {
+        if(resourceHandle != null) {
+            resourceHandle.close();
+            resourceHandle = null;
+        }
+        } catch(Exception e) {
+        System.err.println("Exception closing resource: " + e.getMessage());
+        e.printStackTrace();
+        }
+    }
+    }
+
 }
 

main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/Makefile.in

@@ -15 +15 @@
 ApplyXSLT.jar: $(CLASSES)
     $(JAR) cvf ApplyXSLT.jar -C ../../..  org/nzdl/gsdl/ApplyXSLT.class \
-      -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class
+      -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class \
+      -C ../../.. org/nzdl/gsdl/ApplyXSLT\$$ReadStreamGobbler.class
 
 

main/trunk/greenstone2/runtime-src/src/recpt/cgiwrapper.cpp

@@ -72 +72 @@
 #endif
 
+#if defined(GSDL_USE_GTI_ACTION)
+#include "gtiaction.h"
+#endif
+
 #if defined(USE_RSS)
 #include "rssaction.h"

main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.cpp

@@ -31 +31 @@
 #include <expat.h>
 #include <stdio.h>
+#include <cstring>
 #include <sys/utsname.h>
 #include "gtiaction.h"
@@ -36 +37 @@
 #include "fileutil.h"
 #include "gsdlunicode.h"
+#include "gsdltools.h"
 
 
@@ -85 +87 @@
   text_t set_gsdlos = "GSDLOS=";
 
+  // path to java executable.
+
+  if(java.empty()) {
+    //java = "$JAVA_HOME/bin/java"; // doesn't work
+    //java = "/research/root/jdk1.6.0_27/bin/java"; // can hardcode it, which may be necessary on nzdl or if default java is no longer version 6
+    java = "java";
+  }
+
 #if defined (__WIN32__)
+  path_separator = ";"; // windows (class)PATH separator
+
   set_gsdlos += "windows";
 #else
+  path_separator = ":"; // linux (class)PATH separator
+
   struct utsname *buf = new struct utsname();
   if (uname(buf) == -1) {
@@ -638 +652 @@
   text_t gti_arguments = "";
   if (target_chunk_type == "work") {
-    gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
+
+    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
+
+    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti:/research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl";
+
+    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline:
+    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl
+    gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | " + java + " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";
+
   } else {
-    gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
+    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
+
+    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl";
+
+    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline:
+    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl
+    gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | " + java +  " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";
   }
   

main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.h

@@ -68 +68 @@
   char* set_gsdlos_cstr;
 
+  text_t path_separator;
+  text_t java; // java bin path to preferrably 1.6. Set this here or in init() if the path to java is very specific (such as on nzdl)
+
 public:
   gtiaction ();