Changeset 28973


Ignore:
Timestamp:
2014-04-14T22:20:14+12:00 (10 years ago)
Author:
ak19
Message:

All the changes required to get the GTI installed on a local greenstone using the latest gs2 src code from svn, since this has the security updates. 1. ApplyXSLT in build-src needed a lot of code additions since runtime-src's gtiaction.cpp needs to send the xml file from stdin using a pipe command. ApplyXSLT used to read stdin from a piped cmd differently, expecting DocStart and DocEnd embedding tags to mark the start and stop of each stream. This is still used by code in BasePlugout, so ApplyXSLT has been modified to take a minus-c parameter when requested to read from stdin without special embedding tag markers, such as when gtiaction calls it. ApplyXSLT uses an internal StreamGobbler class to read the stdin since it takes a while for xml generated by the gti.pl (which is piped in) to come in to ApplyXSLT. 2. The inner StreamGobbler class needed to be added into the ApplyXSLT jar file, so the Makefile.in has been updated. 3. In runtime-src, added in missing header files and updated the code that generated the spreadsheets on GTI, since it was firstly hardcoded to use paths on /home/nzdl, and the code that generated the spreadsheets when running ApplyXSLT/xalan.jar no longer worked as it had been coded.

Location:
main/trunk/greenstone2
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/ApplyXSLT.java

    r28709 r28973  
    160160       
    161161      }
     162
     163      //if(br != null) {
     164      //  br.close();
     165      //  br = null;
     166      //}
     167     
    162168    }catch (Exception e)
    163169      {
     
    166172               
    167173    return false;
     174  }
     175 
     176    // reads xml from stdin, but no <?DocStart?><?DocEnd?> markers, and sends output to STDOUT
     177  private boolean processPipedFromStdIn()
     178  {
     179      try{
     180      //System.err.println("Received nothing\n");     
     181     
     182      ReadStreamGobbler readInStreamGobbler = new ReadStreamGobbler(System.in, true);
     183      readInStreamGobbler.start();
     184      readInStreamGobbler.join();
     185      String outputstr = readInStreamGobbler.getOutput();
     186
     187      // Using join() above, even though we use only one streamGobbler thread, and even
     188      // though we're not dealing with the input/output/error streams of a Process object.
     189      // But the join() call here didn't break things.
     190      // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
     191     
     192      boolean result = false;
     193      if (mapping_file !=null && !mapping_file.equals("")){
     194          result = translateXMLWithMapping(outputstr, null); // null: no outputfile, send to STDOUT
     195      }
     196      else{
     197          result = translateXML(outputstr, null); // null: no outputfile, send to STDOUT
     198      }
     199     
     200      if (!result){
     201          System.err.println("Translation Failed!!");
     202          return false;
     203      } else {
     204          return true;   
     205      }
     206      }catch (Exception e) {
     207      System.err.println("Receiving piped data error!" + e.toString());
     208      }
     209      return false;
    168210  }
    169211   
     
    180222    setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT
    181223
    182     transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file)));
     224    if(output_file != null) {
     225    transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file)));
     226    } else {
     227    transformer.transform(new StreamSource(str), new StreamResult(System.out));
     228    }
    183229    return true;
    184230  }
     
    437483    String targetlang="";
    438484
     485    boolean readFromStdInFlag = false;
     486
    439487    HashMap paramMap = new HashMap();
    440488    int index = -1; // index of the '=' sign in cmdline argument specifying custom parameters to be passed into the XSLT
     
    470518      else if(args[i].equals("-l") && i+1 < args.length && !args[i+1].startsWith("-")){
    471519    targetlang = args[++i];     
     520      }
     521      else if(args[i].equals("-c")){
     522      readFromStdInFlag = true;
    472523      }
    473524      else if(args[i].equals("-h")){
     
    489540         
    490541    ApplyXSLT core = null;
    491    
     542
    492543    if (xml_file.equals("") && !xsl_file.equals("")){//read from pipe line
    493544      if (mapping_file.equals("")){
     
    499550           
    500551      if (core != null){
    501     core.process();
     552      if(readFromStdInFlag) { // ApplyXSLT was run with -c: read from pipe but no <?DocStart?><?DocEnd?> markers
     553          core.processPipedFromStdIn();
     554      }
     555    core.process(); //read from pipe line, but expecting <?DocStart?><?DocEnd?> embedding markers
    502556      }
    503557      else{
     
    533587    System.out.println("Usage: ApplyXSLT -x File -t File [-m File] [-o File] [-s sourcelang] [-l targetlang] [param-name=param-value]");
    534588    System.out.println("\t-x specifies the xml file (Note: optional for piped xml data)");
     589    System.out.println("\t-c read xml file piped from stdin but without DocStart/DocEnd markers. Writes to stdout");
    535590    System.out.println("\t-t specifies the xsl file");
    536591    System.out.println("\t-m specifies the mapping file (for MARCXMLPlugout.pm only)");
     
    541596    System.exit(-1);
    542597  }
     598
     599
     600    // StreamGobblers used in reading/writing to Process' input and outputstreams can be re-used more generally.
     601    // Here in ApplyXSTL.java we use it to read from a pipe line (stdin piped into this ApplyXSLT.java)
     602    // Code based on http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
     603    class ReadStreamGobbler extends Thread
     604    {
     605    InputStream is = null;
     606    StringBuffer outputstr = new StringBuffer();
     607    boolean split_newlines = false;
     608   
     609   
     610    public ReadStreamGobbler(InputStream is)
     611    {
     612        this.is = is;
     613        split_newlines = false;
     614    }
     615
     616    public ReadStreamGobbler(InputStream is, boolean split_newlines)
     617    {
     618        this.is = is;
     619        this.split_newlines = split_newlines;
     620    }
     621   
     622    public void run()
     623    {
     624        BufferedReader br = null;
     625        try {
     626        br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
     627        String line=null;
     628        while ( (line = br.readLine()) != null) {
     629            //System.out.println("@@@ GOT LINE: " + line);
     630            outputstr.append(line);
     631            if(split_newlines) {
     632            outputstr.append("\n");
     633            }
     634        }
     635        } catch (IOException ioe) {
     636        ioe.printStackTrace(); 
     637        } finally {
     638        closeResource(br);
     639        }
     640    }
     641   
     642    public String getOutput() {
     643        return outputstr.toString();
     644    }
     645
     646    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html
     647    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks
     648    public void closeResource(Closeable resourceHandle) {
     649        try {
     650        if(resourceHandle != null) {
     651            resourceHandle.close();
     652            resourceHandle = null;
     653        }
     654        } catch(Exception e) {
     655        System.err.println("Exception closing resource: " + e.getMessage());
     656        e.printStackTrace();
     657        }
     658    }
     659    }
     660
    543661}
    544662
  • main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/Makefile.in

    r23356 r28973  
    1515ApplyXSLT.jar: $(CLASSES)
    1616    $(JAR) cvf ApplyXSLT.jar -C ../../..  org/nzdl/gsdl/ApplyXSLT.class \
    17       -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class
     17      -C ../../.. org/nzdl/gsdl/ApplyXSLTUtil.class \
     18      -C ../../.. org/nzdl/gsdl/ApplyXSLT\$$ReadStreamGobbler.class
    1819
    1920
  • main/trunk/greenstone2/runtime-src/src/recpt/cgiwrapper.cpp

    r27172 r28973  
    7272#endif
    7373
     74#if defined(GSDL_USE_GTI_ACTION)
     75#include "gtiaction.h"
     76#endif
     77
    7478#if defined(USE_RSS)
    7579#include "rssaction.h"
  • main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.cpp

    r28913 r28973  
    3131#include <expat.h>
    3232#include <stdio.h>
     33#include <cstring>
    3334#include <sys/utsname.h>
    3435#include "gtiaction.h"
     
    3637#include "fileutil.h"
    3738#include "gsdlunicode.h"
     39#include "gsdltools.h"
    3840
    3941
     
    8587  text_t set_gsdlos = "GSDLOS=";
    8688
     89  // path to java executable.
     90
     91  if(java.empty()) {
     92    //java = "$JAVA_HOME/bin/java"; // doesn't work
     93    //java = "/research/root/jdk1.6.0_27/bin/java"; // can hardcode it, which may be necessary on nzdl or if default java is no longer version 6
     94    java = "java";
     95  }
     96
    8797#if defined (__WIN32__)
     98  path_separator = ";"; // windows (class)PATH separator
     99
    88100  set_gsdlos += "windows";
    89101#else
     102  path_separator = ":"; // linux (class)PATH separator
     103
    90104  struct utsname *buf = new struct utsname();
    91105  if (uname(buf) == -1) {
     
    638652  text_t gti_arguments = "";
    639653  if (target_chunk_type == "work") {
    640     gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
     654
     655    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
     656
     657    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti:/research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl";
     658
     659    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline:
     660    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl
     661    gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | " + java + " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";
     662
    641663  } else {
    642     gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
     664    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";
     665
     666    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /research/root/jdk1.6.0_27/bin/java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl";
     667
     668    // don't need to add toplevel gsdlhome to classpath actually, as the following will work from the commandline:
     669    // gti.pl get-all-chunks nl coredm | java -cp /research/ak19/gs2-svn-gti/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -t /research/ak19/gs2-svn-gti/bin/script/gti-generate-excel-xml.xsl
     670    gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | " + java +  " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";
    643671  }
    644672 
  • main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.h

    r18460 r28973  
    6868  char* set_gsdlos_cstr;
    6969
     70  text_t path_separator;
     71  text_t java; // java bin path to preferrably 1.6. Set this here or in init() if the path to java is very specific (such as on nzdl)
     72
    7073public:
    7174  gtiaction ();
Note: See TracChangeset for help on using the changeset viewer.