source: main/trunk/greenstone2/build-src/src/java/org/nzdl/gsdl/ApplyXSLT.java@ 38850

/**********************************************************************
 *
 * ApplyXSLT.java
 *
 * Copyright 2006-2010  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/


package org.nzdl.gsdl;

import java.io.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;

import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;

import javax.xml.parsers.*;
import javax.xml.transform.dom.*;
import org.w3c.dom.*;



/**
 *  Use the TraX interface to perform a transformation in the simplest manner possible
 *  (3 statements).
 */
public class ApplyXSLT
{

  public static final String DOC_START = new String ("<?DocStart?>"); 
  public static final String DOC_END = new String ("<?DocEnd?>"); 
  public static final String INPUT_END = new String ("<?Done?>"); 

  private static final String RECORD_ELEMENT = "record";
  private static final String CONTROLFIELD_ELEMENT = "controlfield";
  private static final String SUBFIELD_ELEMENT = "subfield";
  private static final String LEADER_ELEMENT = "leader";

  private final int BEFORE_READING = 0;
  private final int IS_READING = 1;
  private String xsl_file;
  private String mapping_file;

    private String sourcelang;
    private String targetlang;
    private HashMap paramMap;

    public ApplyXSLT(String sourcelang, String targetlang, HashMap param_map){
    initParams(sourcelang, targetlang, param_map);
    }
   
    public ApplyXSLT(String xsl_file, String sourcelang, String targetlang, HashMap param_map)
  {
    this.xsl_file = xsl_file; 
    initParams(sourcelang, targetlang, param_map);
  }

    public ApplyXSLT(String xsl_file, String mapping_file, String sourcelang, String targetlang, HashMap param_map) {
    this.xsl_file = xsl_file; 
    this.mapping_file = mapping_file;
    initParams(sourcelang, targetlang, param_map);
  }

    private void initParams(String sourcelang, String targetlang, HashMap param_map)
    {
    this.sourcelang = sourcelang;
    this.targetlang = targetlang;
    // if only target language is provided, assume source language is English
    if(sourcelang.equals("") && !targetlang.equals("")) {
        this.sourcelang = "en";
    }

    // any custom parameters to be passed into the XSLT would be in the map by now
    paramMap = param_map;
    }

    /*
    private boolean isOpen(BufferedReader br) {
    // The code used to rely on br.ready() to determine whether it should
    // continue looping to read input (from standard-in).  However on
    // closer read of the JavaDoc, br.ready() can return false even
    // when the input stream is still open -- it's just that there isn't
    // any input lines to read in at the moment.

    // If the input stream has been closed (from the external calling Perl code),
    // then the only sure way to determine that this condition has arise is
    // to ask br.ready() and catch the Exception that occurs ... because the
    // input stream is not closed!

    boolean is_open = true;
    
    try {
        boolean is_ready = br.ready();
    }
    catch (Exception e) {  // Could be more selective and go for IOException ??
        //System.err.println("ApplyXSLT::isOpen() encountered an exception => so input is closed");
        is_open = false;
    }

    return is_open;
    }
    */
    
  private boolean process()
  {
    try{

      // Use System InputStream to receive piped data from the perl program
      InputStreamReader ir = new InputStreamReader(System.in, "UTF8");
      BufferedReader br = new BufferedReader(ir);
    
      int system_status = BEFORE_READING;
      StringBuffer a_doc = new StringBuffer();
      String output_file = new String();


      String this_line;      
      while ((this_line = br.readLine()) != null) {
        
    //System.err.println("Read in from pipe, line: " + this_line);
    
    if(system_status == BEFORE_READING){
      if(this_line.compareTo(DOC_START) == 0){
          // If this_line is DOC_START then we require the next line of input to be
          // the filename
        output_file = br.readLine(); // read the next line as the output file name
        if (output_file == null) {
        // A problem of some form occurred
        return false;
        }
        //System.err.println("Read in from pipe, next line: " + output_file);
        system_status = IS_READING;
        a_doc = new StringBuffer();
      }
      else if(this_line.compareTo(INPUT_END) == 0){
        return true;
      }
      else{
        system_status = BEFORE_READING;
      }
            
    }
    else if(system_status == IS_READING){
      if(this_line.compareTo(DOC_END) == 0){
        boolean result = false;
        if (mapping_file !=null && !mapping_file.equals("")){
        result = translateXMLWithMapping(a_doc.toString(), output_file);
        }
        else{
        result = translateXML(a_doc.toString(), output_file);
        }
            
        if (!result){
          return false;
        }
            
        system_status = BEFORE_READING;
            
      }
      else{
          a_doc.append(this_line + "\n");
      }
    }
    else{         
      System.err.println ("Undefined system status in ApplyXSLT.java main().");
      System.exit(-1);
    }
        
      }

      return true;
      
      //if(br != null) {
      //  br.close();
      //  br = null;
      //}      
    }
    catch (Exception e) {
    System.err.println("Receiving piped data error!" + e.toString());
    e.printStackTrace();
    }
                
    return false;
  }
  
    // reads xml from stdin, but no <?DocStart?><?DocEnd?> markers, and sends output to STDOUT
  private boolean processPipedFromStdIn()
  {
      try{
      //System.err.println("Received nothing\n");      
      
      ReadStreamGobbler readInStreamGobbler = new ReadStreamGobbler(System.in, true);
      readInStreamGobbler.start();
      readInStreamGobbler.join();
      String outputstr = readInStreamGobbler.getOutput();

      // Using join() above, even though we use only one streamGobbler thread, and even
      // though we're not dealing with the input/output/error streams of a Process object.
      // But the join() call here didn't break things.
      // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
      
      boolean result = false;
      if (mapping_file !=null && !mapping_file.equals("")){
          result = translateXMLWithMapping(outputstr, null); // null: no outputfile, send to STDOUT
      }
      else{
          result = translateXML(outputstr, null); // null: no outputfile, send to STDOUT
      }
      
      if (!result){
          System.err.println("Translation Failed!!");
          return false;
      } else {
          return true;   
      }
      }catch (Exception e) {
      System.err.println("Receiving piped data error!" + e.toString());
      e.printStackTrace();
      } 
      return false;
  }
    
    
  private boolean translateXML(String full_doc, String output_file) 
    throws IOException,TransformerException, TransformerConfigurationException, FileNotFoundException
  {
    
    StringReader str = new StringReader(full_doc) ;
    
    TransformerFactory tFactory = TransformerFactory.newInstance();
    Transformer transformer = tFactory.newTransformer(new StreamSource(xsl_file));

    setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT

    if(output_file != null) {
    transformer.transform(new StreamSource(str), new StreamResult(new FileOutputStream(output_file)));
    } else {
    transformer.transform(new StreamSource(str), new StreamResult(System.out));
    }
    return true;
  }

  private boolean translateXMLWithMapping(String full_doc, String output_file) 
    throws IOException,TransformerException, TransformerConfigurationException, FileNotFoundException
  { 
    StringReader str = new StringReader(full_doc) ;

    try{
      TransformerFactory tFactory = TransformerFactory.newInstance();
      Transformer transformer = tFactory.newTransformer(new StreamSource(xsl_file));
        
      Document mapping_doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(mapping_file);
      Element mapping =mapping_doc.getDocumentElement(); 
        
      transformer.setParameter("mapping",mapping);
      setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT
        
      Document output_doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
        
      transformer.transform(new StreamSource(str), new DOMResult(output_doc));
        
      calculateRecordsLength(output_doc);
       
      transformer = tFactory.newTransformer();
       
      transformer.transform(new DOMSource(output_doc), new StreamResult(new FileOutputStream(output_file)));

    }
    catch(Exception e){
      e.printStackTrace();
      return false;
    }

    return true;
  }

  private void calculateRecordsLength(Document output_doc){
    NodeList records = output_doc.getDocumentElement().getElementsByTagName(RECORD_ELEMENT);

    for(int i=0;i<records.getLength();i++){
      Element record = (Element)records.item(i);
      calculateRecordLength(record);
    }
  }

  private void calculateRecordLength(Element record){
    int total_length =0;    
    NodeList controlfileds = record.getElementsByTagName(CONTROLFIELD_ELEMENT);
    for(int i=0;i<controlfileds.getLength();i++){
      Element controlfiled = (Element)controlfileds.item(i);
      total_length +=getElementTextValue(controlfiled).length();
    }
    
    NodeList subfileds = record.getElementsByTagName(SUBFIELD_ELEMENT);
    for(int i=0;i<subfileds.getLength();i++){
      Element subfiled = (Element)subfileds.item(i);
      total_length +=getElementTextValue(subfiled).length();
    }

    String record_length = total_length+"";
    //fill in a extra digit as record length needs to be five characters long
    if (total_length < 10000){
      record_length = "0"+record_length;
      if (total_length < 1000){
    record_length = "0"+record_length;  
      }
      if (total_length < 100){
    record_length = "0"+record_length;  
      }
      if (total_length < 10){
    record_length = "0"+record_length;  
      }
       
    }

    NodeList leaders = record.getElementsByTagName(LEADER_ELEMENT); 
    
    //only one leader element
    if (leaders.getLength() >0){
      Element leader_element = (Element)leaders.item(0);
      removeFirstTextNode(leader_element);
      leader_element.insertBefore(leader_element.getOwnerDocument().createTextNode(record_length),leader_element.getFirstChild()); 
    }

  }

  private void removeFirstTextNode(Element element){
    //remove the first text node
    NodeList children_nodelist = element.getChildNodes();
    for (int i = 0; i < children_nodelist.getLength(); i++) {
      Node child_node = children_nodelist.item(i);
      if (child_node.getNodeType() == Node.TEXT_NODE) {
    element.removeChild(child_node);
    return;
      }
    }

  }

  private String getElementTextValue(Element element)
  {
    String text ="";    

    // Find the node child
    NodeList children_nodelist = element.getChildNodes();
    for (int i = 0; i < children_nodelist.getLength(); i++) {
      Node child_node = children_nodelist.item(i);
      if (child_node.getNodeType() == Node.TEXT_NODE) {
    text +=child_node.getNodeValue();
      }
    }

    return text;
  }
 

  private void setMappingVariable(Document style_doc){
    Node child = style_doc.getDocumentElement().getFirstChild();
    while(child != null) {
      String name = child.getNodeName();
      if (name.equals("xsl:variable")) {
    Element variable_element = (Element)child;
    if ( variable_element.getAttribute("name").trim().equals("mapping")){
      variable_element.setAttribute("select","document('"+mapping_file+"')/Mapping");
    }
      }
      child = child.getNextSibling();
    }
    
  } 

    private void  setTransformerParams(Transformer transformer) 
    {
    if(targetlang != "") {
        transformer.setParameter("sourcelang",sourcelang);
        transformer.setParameter("targetlang",targetlang);
    }

    // handle any custom parameters that are also to be passed into the XSLT
    Iterator i = paramMap.entrySet().iterator(); 
    while(i.hasNext()) {
        Map.Entry entry = (Map.Entry)i.next();
        String paramName = (String)entry.getKey();
        String paramValue = (String)entry.getValue();

        transformer.setParameter(paramName, paramValue);
    }
    
    }

  private void translate(String xml_file, String xsl_file, String output_file)throws IOException,TransformerException, TransformerConfigurationException, FileNotFoundException, IOException{             
         
    TransformerFactory tFactory = TransformerFactory.newInstance();
    Transformer transformer = tFactory.newTransformer(new StreamSource(xsl_file));
           
    OutputStreamWriter output = null;  
    if (output_file.equals("")) {
      output = new OutputStreamWriter(System.out, "UTF-8");
    }
    else{
      output = new OutputStreamWriter(new FileOutputStream(output_file), "UTF-8"); 
    }

    setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT
    transformer.transform(new StreamSource(new File(xml_file)),new StreamResult(output));
        
  }
  private void translateWithMapping(String xml_file, String xsl_file, String mapping_file, String output_file)throws IOException,TransformerException, TransformerConfigurationException, FileNotFoundException {             
         
    TransformerFactory tFactory = TransformerFactory.newInstance();
    Transformer transformer = tFactory.newTransformer(new StreamSource(xsl_file));
           
    OutputStreamWriter output = null;  
    if (output_file.equals("")) {
      output = new OutputStreamWriter(System.out, "UTF-8");
    }
    else{
      output = new OutputStreamWriter(new FileOutputStream(output_file), "UTF-8"); 
    }
    try {
      Document mapping_doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(mapping_file);
      Element mapping =mapping_doc.getDocumentElement(); 
        
      transformer.setParameter("mapping",mapping);
    } catch (Exception e) {
      System.err.println("Couldn't load in mapping file");
      e.printStackTrace();
    }
    setTransformerParams(transformer); // sourcelang and targetlang and any further custom parameters to be passed into the XSLT
    transformer.transform(new StreamSource(new File(xml_file)),new StreamResult(output));
        
  }

  static public String replaceAll(String source_string, String match_regexp, String replace_string)
  {
    return source_string.replaceAll(match_regexp, replace_string);
  }

    // Necessary for paperspast.dm, but can be used generally. 
    // The get-chunks cmd of gti.pl perl script when run over paperspast.dm returns XML with source and target lines 
    // like: [c=paperspast] {All newspapers} for source and [c=paperspast,l=mi] {Niupepa katoa} for target
    // This function returns just the 'string' portion of the chunk of data: e.g 'All newspapers' and 'Niupepa katoa'
    static public String getChunkString(String target_file_text)
    {
    int startindex = target_file_text.indexOf("[");
    if(startindex != 0) {
        return target_file_text;
    } // to test that the input requires processing

    // else
    startindex = target_file_text.indexOf("{");
    int endindex = target_file_text.lastIndexOf("}");
    if(startindex != -1 && endindex != -1) {
        return target_file_text.substring(startindex+1, endindex); // skips { and }
    } else {
        return target_file_text;
    }
    
    }

    // Necessary for paperspast.dm, but can be used generally. 
    // The get-chunks cmd of gti.pl perl script when run over paperspast.dm returns XML with source and target lines 
    // like: [c=paperspast] {All newspapers} for source and [c=paperspast,l=mi] {Niupepa katoa} for target
    // This function returns just the 'attribute' portion of the chunk of data: e.g 'c=paperspast' and 'c=paperspast,l=mi'
    static public String getChunkAttr(String target_file_text)
    {
    int startindex = target_file_text.indexOf("[");
    if(startindex != 0) {
        return target_file_text;
    } // to test that the input requires processing

    // else
    startindex = target_file_text.indexOf("{");
    int endindex = target_file_text.lastIndexOf("}");
    if(startindex != -1 && endindex != -1) {
        endindex = target_file_text.lastIndexOf("]", startindex); // look for ] preceding the {
        if(endindex > 1) { //if(endindex != -1) {
                       // so there's something to substring between [ and ]                        
        return target_file_text.substring(1, endindex).trim(); // skips [ and ]
        }
    } 
    return target_file_text;
    }

  public static void main(String[] args) 
  { 
    String xml_file="";
    String xsl_file="";
    String mapping_file="";
    String output_file="";

    String sourcelang="";
    String targetlang="";

    boolean readFromStdInFlag = false;

    HashMap paramMap = new HashMap();
    int index = -1; // index of the '=' sign in cmdline argument specifying custom parameters to be passed into the XSLT

    // Checking Arguments
    if(args.length < 1)
      {
    printUsage(); 
      }           

    for (int i=0;i<args.length;i++){
      if (args[i].equals("-m") && i+1 < args.length && !args[i+1].startsWith("-")){
    mapping_file = args[++i];
    checkFile(mapping_file.replaceAll("file:///",""));
      }
      else if (args[i].equals("-x") && i+1 < args.length && !args[i+1].startsWith("-")){
    xml_file = args[++i]; 
    checkFile(xml_file.replaceAll("file:///",""));
      }
      else if(args[i].equals("-t") && i+1 < args.length && !args[i+1].startsWith("-")){
    xsl_file = args[++i];
    checkFile( xsl_file.replaceAll("file:///","")); 
      }
      else if(args[i].equals("-o") && i+1 < args.length && !args[i+1].startsWith("-")){
    output_file = args[++i];
        
      }
      // The two language parameters (-s and -l) are for the gti-generate-tmx-xml file 
      // which requires the target lang (code), and will accept the optional source lang (code)
      else if(args[i].equals("-s") && i+1 < args.length && !args[i+1].startsWith("-")){
    sourcelang = args[++i];     
      }
      else if(args[i].equals("-l") && i+1 < args.length && !args[i+1].startsWith("-")){
    targetlang = args[++i];     
      }
      else if(args[i].equals("-c")){
      readFromStdInFlag = true;
      }
      else if(args[i].equals("-h")){
    printUsage();
      }
      else if ((index = args[i].indexOf("=")) != -1) { // custom parameters provided on the cmdline in the form paramName1=paramValue1 paramName2=paramValue2 etc 
      // that are to be passed into the XSLT
      String paramName = args[i].substring(0, index);
      String paramValue = args[i].substring(index+1); // skip the = sign
      paramMap.put(paramName, paramValue);
      index = -1;
      }
      else{
    printUsage();
      }
      
    }

         
    ApplyXSLT core = null;

    if (xml_file.equals("") && !xsl_file.equals("")){//read from pipe line
      if (mapping_file.equals("")){ 
      core = new ApplyXSLT(xsl_file, sourcelang, targetlang, paramMap);
      }
      else{
      core = new ApplyXSLT(xsl_file, mapping_file, sourcelang, targetlang, paramMap);  
      }
            
      if (core != null){
      if(readFromStdInFlag) { // ApplyXSLT was run with -c: read from pipe but no <?DocStart?><?DocEnd?> markers
          core.processPipedFromStdIn();
      }
    core.process(); //read from pipe line, but expecting <?DocStart?><?DocEnd?> embedding markers
      }
      else{
    printUsage(); 
      }
    }
    else if(!xml_file.equals("") && !xsl_file.equals("")){
    core = new ApplyXSLT(sourcelang, targetlang, paramMap);
      try {
    if (mapping_file.equals("")) {
      core.translate(xml_file,xsl_file,output_file); 
    } else {
      core.translateWithMapping(xml_file,xsl_file,mapping_file, output_file); 
    }  
      }
      catch(Exception e){e.printStackTrace();}
    } 
    else{
      printUsage();    
    }
    
  }

  private static void checkFile(String filename){
    File file = new File(filename);
    if (!file.exists()){
      System.out.println("Error: "+filename+" doesn't exist!");
      System.exit(-1);
    }
  }
  
  private  static void printUsage(){
    System.out.println("Usage: ApplyXSLT -x File -t File [-m File] [-o File] [-s sourcelang] [-l targetlang] [param-name=param-value]");
    System.out.println("\t-x specifies the xml file (Note: optional for piped xml data)");
    System.out.println("\t-c read xml file piped from stdin but without DocStart/DocEnd markers. Writes to stdout");
    System.out.println("\t-t specifies the xsl file");
    System.out.println("\t-m specifies the mapping file (for MARCXMLPlugout.pm only)");
    System.out.println("\t-o specifies the output file name (output to screen if this option is absent)");
    System.out.println("\t-s specifies the input language code for generating TMX file. Defaults to 'en' if none is provided");
    System.out.println("\t-l specifies the output language code. Required if generating a TMX file.");
    System.out.println("\tFor general transformations of an XML by an XSLT, you can pass in parameter name=value pairs if any need to passed on into the XSLT as xsl params.");
    System.exit(-1);
  }


    // StreamGobblers used in reading/writing to Process' input and outputstreams can be re-used more generally.
    // Here in ApplyXSTL.java we use it to read from a pipe line (stdin piped into this ApplyXSLT.java)
    // Code based on http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
    class ReadStreamGobbler extends Thread
    {
    InputStream is = null;
    StringBuffer outputstr = new StringBuffer();
    boolean split_newlines = false;
    
    
    public ReadStreamGobbler(InputStream is)
    {
        this.is = is;
        split_newlines = false;
    }

    public ReadStreamGobbler(InputStream is, boolean split_newlines)
    {
        this.is = is;
        this.split_newlines = split_newlines;
    }
    
    public void run()
    {
        BufferedReader br = null;
        try {
        br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
        String line=null;
        while ( (line = br.readLine()) != null) {
            //System.out.println("@@@ GOT LINE: " + line);
            outputstr.append(line);
            if(split_newlines) {
            outputstr.append("\n");
            }
        }
        } catch (IOException ioe) {
        ioe.printStackTrace();  
        } finally {
        System.err.println("ReadStreamGobbler:run() finished.  Closing resource");
        closeResource(br);
        }
    }
    
    public String getOutput() { 
        return outputstr.toString(); 
    }

    // http://docs.oracle.com/javase/tutorial/essential/exceptions/finally.html
    // http://stackoverflow.com/questions/481446/throws-exception-in-finally-blocks
    public void closeResource(Closeable resourceHandle) {
        try {
        if(resourceHandle != null) {
            resourceHandle.close();
            resourceHandle = null;
        }
        } catch(Exception e) {
        System.err.println("Exception closing resource: " + e.getMessage());
        e.printStackTrace();
        }
    }
    }

}