Changeset 28973

Show
Ignore:
Timestamp:
14.04.2014 22:20:14 (5 years ago)
Author:
ak19
Message:

All the changes required to get the GTI installed on a local greenstone using the latest gs2 src code from svn, since this has the security updates. 1. ApplyXSLT in build-src needed a lot of code additions since runtime-src's gtiaction.cpp needs to send the xml file from stdin using a pipe command. ApplyXSLT used to read stdin from a piped cmd differently, expecting DocStart? and DocEnd? embedding tags to mark the start and stop of each stream. This is still used by code in BasePlugout?, so ApplyXSLT has been modified to take a minus-c parameter when requested to read from stdin without special embedding tag markers, such as when gtiaction calls it. ApplyXSLT uses an internal StreamGobbler? class to read the stdin since it takes a while for xml generated by the gti.pl (which is piped in) to come in to ApplyXSLT. 2. The inner StreamGobbler? class needed to be added into the ApplyXSLT jar file, so the Makefile.in has been updated. 3. In runtime-src, added in missing header files and updated the code that generated the spreadsheets on GTI, since it was firstly hardcoded to use paths on /home/nzdl, and the code that generated the spreadsheets when running ApplyXSLT/xalan.jar no longer worked as it had been coded.

Location:
main/trunk/greenstone2
Files:
5 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/ApplyXSLT.java

    r28709 r28973  
    160160         
    161161      } 
     162 
     163      //if(br != null) { 
     164      //  br.close(); 
     165      //  br = null; 
     166      //} 
     167       
    162168    }catch (Exception e) 
    163169      { 
     
    166172                 
    167173    return false; 
     174  } 
     175   
     176    // reads xml from stdin, but no <?DocStart?><?DocEnd?> markers, and sends output to STDOUT 
     177  private boolean processPipedFromStdIn() 
     178  { 
     179      try{ 
     180      //System.err.println("Received nothing\n");       
     181       
     182      ReadStreamGobbler readInStreamGobbler = new ReadStreamGobbler(System.in, true); 
     183      readInStreamGobbler.start(); 
     184      readInStreamGobbler.join(); 
     185      String outputstr = readInStreamGobbler.getOutput(); 
     186 
     187      // Using join() above, even though we use only one streamGobbler thread, and even 
     188      // though we're not dealing with the input/output/error streams of a Process object. 
     189      // But the join() call here didn't break things. 
     190      // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2 
     191       
     192      boolean result = false; 
     193      if (mapping_file !=null && !mapping_file.equals("")){ 
     194          result = translateXMLWithMapping(outputstr, null); // null: no outputfile, send to STDOUT 
     195      } 
     196      else{ 
     197          result = translateXML(outputstr, null); // null: no outputfile, send to STDOUT 
     198      } 
     199       
     200      if (!result){ 
     201          System.err.println("Translation Failed!!"); 
     202          return false; 
     203      } else { 
     204          return true;    
     205      } 
     206      }catch (Exception e) { 
     207      System.err.println("Receiving piped data error!" + e.toString()); 
     208      }  
     209      return false; 
    168210  } 
    169211     
     
    180222    setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT 
    181223 
    182     transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file))); 
     224    if(output_file != null) { 
     225    transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file))); 
     226    } else { 
     227    transformer.transform(new StreamSource(str), new StreamResult(System.out)); 
     228    } 
    183229    return true; 
    184230  } 
     
    437483    String targetlang=""; 
    438484 
     485    boolean readFromStdInFlag = false; 
     486 
    439487    HashMap paramMap = new HashMap(); 
    440488    int index = -1; // index of the '=' sign in cmdline argument specifying custom parameters to be passed into the XSLT 
     
    470518      else if(args[i].equals("-l") && i+1 < args.length && !args[i+1].startsWith("-")){ 
    471519    targetlang = args[++i];      
     520      } 
     521      else if(args[i].equals("-c")){ 
     522      readFromStdInFlag = true; 
    472523      } 
    473524      else if(args[i].equals("-h")){ 
     
    489540          
    490541    ApplyXSLT core = null; 
    491      
     542 
    492543    if (xml_file.equals("") && !xsl_file.equals("")){//read from pipe line 
    493544      if (mapping_file.equals("")){  
     
    499550             
    500551      if (core != null){ 
    501     core.process(); 
     552      if(readFromStdInFlag) { // ApplyXSLT was run with -c: read from pipe but no <?DocStart?><?DocEnd?> markers 
     553          core.processPipedFromStdIn(); 
     554      } 
     555    core.process(); //read from pipe line, but expecting <?DocStart?><?DocEnd?> embedding markers 
    502556      } 
    503557      else{ 
     
    533587    System.out.println("Usage: ApplyXSLT -x File -t File [-m File] [-o File] [-s sourcelang] [-l targetlang] [param-name=param-value]"); 
    534588    System.out.println("\t-x specifies the xml file (Note: optional for piped xml data)"); 
     589    System.out.println("\t-c read xml file piped from stdin but without DocStart/DocEnd markers. Writes to stdout"); 
    535590    System.out.println("\t-t specifies the xsl file"); 
    536591    System.out.println("\t-m specifies the mapping file (for MARCXMLPlugout.pm only)"); 
     
    541596    System.exit(-1); 
    542597  } 
     598 
     599 
     600    // StreamGobblers used in reading/writing to Process' input and outputstreams can be re-used more generally. 
     601    // Here in ApplyXSTL.java we use it to read from a pipe line (stdin piped into this ApplyXSLT.java) 
     602    // Code based on http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2 
     603    class ReadStreamGobbler extends Thread 
     604    { 
     605    InputStream is = null; 
     606    StringBuffer outputstr = new StringBuffer(); 
     607    boolean split_newlines = false; 
     608     
     609     
     610    public ReadStreamGobbler(InputStream is) 
     611    { 
     612        this.is = is; 
     613        split_newlines = false; 
     614    } 
     615 
     616    public ReadStreamGobbler(InputStream is, boolean split_newlines) 
     617    { 
     618        this.is = is; 
     619        this.split_newlines = split_newlines; 
     620    } 
     621     
     622    public void run() 
     623    { 
     624        BufferedReader br = null; 
     625        try { 
     626        br = new BufferedReader(new InputStreamReader(is, "UTF-8")); 
     627        String line=null; 
     628        while ( (line = br.readLine()) != null) { 
     629            //System.out.println("@@@ GOT LINE: " + line); 
     630            outputstr.append(line); 
     631            if(split_newlines) { 
     632            outputstr.append("\n"); 
     633            } 
     634        } 
     635        } catch (IOException ioe) { 
     636        ioe.printStackTrace();   
     637        } finally { 
     638        closeResource(br); 
     639        } 
     640    } 
     641     
     642    public String getOutput() {  
     643        return outputstr.toString();  
     644    } 
     645 
     646    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html 
     647    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks 
     648    public void closeResource(Closeable resourceHandle) { 
     649        try { 
     650        if(resourceHandle != null) { 
     651            resourceHandle.close(); 
     652            resourceHandle = null; 
     653        } 
     654        } catch(Exception e) { 
     655        System.err.println("Exception closing resource: " + e.getMessage()); 
     656        e.printStackTrace(); 
     657        } 
     658    } 
     659    } 
     660 
    543661} 
    544662 
  • main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/Makefile.in

    r23356 r28973  
    1515ApplyXSLT.jar: $(CLASSES) 
    1616    $(JAR) cvf ApplyXSLT.jar -C ../../..  org/nzdl/gsdl/ApplyXSLT.class \ 
    17       -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class 
     17      -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class \ 
     18      -C ../../.. org/nzdl/gsdl/ApplyXSLT\$$ReadStreamGobbler.class 
    1819 
    1920 
  • main/trunk/greenstone2/runtime-src/src/recpt/cgiwrapper.cpp

    r27172 r28973  
    7272#endif 
    7373 
     74#if defined(GSDL_USE_GTI_ACTION) 
     75#include "gtiaction.h" 
     76#endif 
     77 
    7478#if defined(USE_RSS) 
    7579#include "rssaction.h" 
  • main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.cpp

    r28913 r28973  
    3131#include <expat.h> 
    3232#include <stdio.h> 
     33#include <cstring> 
    3334#include <sys/utsname.h> 
    3435#include "gtiaction.h" 
     
    3637#include "fileutil.h" 
    3738#include "gsdlunicode.h" 
     39#include "gsdltools.h" 
    3840 
    3941 
     
    8587  text_t set_gsdlos = "GSDLOS="; 
    8688 
     89  // path to java executable. 
     90 
     91  if(java.empty()) { 
     92    //java = "$JAVA_HOME/bin/java"; // doesn't work 
     93    //java = "/research/root/jdk1.6.0_27/bin/java"; // can hardcode it, which may be necessary on nzdl or if default java is no longer version 6 
     94    java = "java"; 
     95  } 
     96 
    8797#if defined (__WIN32__) 
     98  path_separator = ";"; // windows (class)PATH separator 
     99 
    88100  set_gsdlos += "windows"; 
    89101#else 
     102  path_separator = ":"; // linux (class)PATH separator 
     103 
    90104  struct utsname *buf = new struct utsname(); 
    91105  if (uname(buf) == -1) { 
     
    638652  text_t gti_arguments = ""; 
    639653  if (target_chunk_type == "work") { 
    640     gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -"; 
     654 
     655    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -"; 
     656 
     657    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti:/research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl"; 
     658 
     659    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline: 
     660    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl 
     661    gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | " + java + " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl"; 
     662 
    641663  } else { 
    642     gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -"; 
     664    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -"; 
     665 
     666    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl"; 
     667 
     668    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline: 
     669    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl 
     670    gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | " + java +  " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl"; 
    643671  } 
    644672   
  • main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.h

    r18460 r28973  
    6868  char* set_gsdlos_cstr; 
    6969 
     70  text_t path_separator; 
     71  text_t java; // java bin path to preferrably 1.6. Set this here or in init() if the path to java is very specific (such as on nzdl) 
     72 
    7073public: 
    7174  gtiaction ();