source: main/trunk/greenstone2/runtime-src/src/recpt/gtiaction.cpp@ 30620

/**********************************************************************
 *
 * gtiaction.cpp -- 
 * Copyright (C) 2005  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/


#include "gsdl_modules_cfg.h"
#ifdef GSDL_USE_GTI_ACTION


#include <expat.h>
#include <stdio.h>
#include <cstring>
#include <sys/utsname.h>
#include "gtiaction.h"
#include "cgiutils.h"
#include "fileutil.h"
#include "gsdlunicode.h"
#include "gsdltools.h"



gtiaction::gtiaction()
{
  cgiarginfo arg_ainfo;

  arg_ainfo.shortname = "tlc";
  arg_ainfo.longname = "translation target language code";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (&cerr, arg_ainfo);

  arg_ainfo.shortname = "tfk";
  arg_ainfo.longname = "translation file key";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (&cerr, arg_ainfo);

  arg_ainfo.shortname = "ncpp";
  arg_ainfo.longname = "number of chunks per page";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "1";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (&cerr, arg_ainfo);
}



gtiaction::~gtiaction()
{
  delete[] set_gsdlhome_cstr;
  delete[] set_gsdlos_cstr;
}



bool gtiaction::init (ostream& /*logout*/)
{
  // Set GSDLHOME and GSDLOS environment variables
  text_t set_gsdlhome = "GSDLHOME=" + gsdlhome;
  text_t set_gsdlos = "GSDLOS=";

  // Path to java executable. If javahome property was set in gsdlsite.cfg, 
  // then common-src/src/lib/gsdlsite.cfg would have read this in and set it in the env
  // If the env var is empty, default to trying to use just "java"
  char* existing_javahome = getenv("JAVA_HOME"); // return value may not be modified by the program

  if(existing_javahome == NULL) {
    //java = "$JAVA_HOME/bin/java"; // doesn't work
    java = "java"; // works locally, but not on nzdl. On nzdl need to set javahome in gsdlsite.cfg
  } else {    
    java = existing_javahome; 
    java += "/bin/java";
  }

#if defined (__WIN32__)
  path_separator = ";"; // windows (class)PATH separator

  set_gsdlos += "windows";
#else
  path_separator = ":"; // linux (class)PATH separator

  struct utsname *buf = new struct utsname();
  if (uname(buf) == -1) {
    // uname failed, so this must be linux
    set_gsdlos += "linux";
  }
  else {
    text_t gsdlos = buf->sysname;
    lc(gsdlos);
    set_gsdlos += gsdlos;
  }
  delete buf;
#endif

  // These will be cleaned up in the destructor
  set_gsdlhome_cstr = set_gsdlhome.getcstr();
  set_gsdlos_cstr = set_gsdlos.getcstr();
  putenv(set_gsdlhome_cstr);
  putenv(set_gsdlos_cstr);

  return true;
}



bool gtiaction::check_cgiargs(cgiargsinfoclass& /*argsinfo*/, cgiargsclass& args,
                  recptprotolistclass* /*protos*/, ostream& logout)
{
  // Authenticate the user, except for the "home" and "lang" pages
  if (args["p"] != "home" && args["p"] != "lang" && args["p"] != "status") {
    args["uan"] = 1;
    args["ug"] = "langadmin_" + args["tlc"];
  }

  return true;
}



bool gtiaction::do_action(cgiargsclass& args, recptprotolistclass* /*protos*/, 
              browsermapclass* /*browsers*/, displayclass& disp, 
              outconvertclass& outconvert, ostream& textout, 
              ostream& logout)
{
  // Special case for producing Excel spreadsheets, as these are downloaded
  if (args["p"] == "excel") {
    return produce_excel_spreadsheet(args, logout);
  }

  textout << outconvert << disp << ("_gti:header_\n") << ("_gti:content_\n") << ("_gti:footer_\n");
  return true;
}



void gtiaction::get_cgihead_info(cgiargsclass& args, recptprotolistclass* /*protos*/,
                 response_t& response, text_t& response_data, 
                 ostream& logout)
{
  // Special case for producing Excel spreadsheets, as these are downloaded
  if (args["p"] == "excel") {
    printf("Content-Disposition: attachment; filename=\"Greenstone-%s-%s.xml\"\n", args["tlc"].getcstr(), args["tfk"].getcstr());
    response = content;
    response_data = "text/xml";
    return;
  }

  response = content;
  response_data = "text/html";
}



void gtiaction::define_internal_macros(displayclass& disp, cgiargsclass& args, 
                       recptprotolistclass* protos, ostream& logout)
{
  // logout << endl << "Arguments: " << args << endl;
  logout << endl << "CGI arg p: " << args["p"] << endl;

  // For some reason this must be done as well as setting the macro in gti.dm
  // is that still true??
  disp.setmacro("preflink", displayclass::defaultpackage, "_gti:preflink_");
  
  // Define the page content for the GTI home page
  if (args["p"] == "home") {
    define_gti_home_page(disp, args, logout);
    return;
  }

  // Define the page content for the GTI language page
  if (args["p"] == "lang") {
    define_gti_lang_page(disp, args, logout);
    return;
  }

  // Define the page content for the GTI search page
  if (args["p"] == "find") {
    define_gti_find_page(disp, args, logout);
    return;
  }

  // Define the page content for the GTI offline page
  if (args["p"] == "offline") {
    define_gti_offline_page(disp, args, logout);
    return;
  }
  
  // Define the page content for the GTI view status page
  if (args["p"] == "status") {
    define_gti_status_page(disp, args, logout);
    return;
  }

  // Process user translations
  if (args["p"] == "submit") {
    process_gti_submissions(disp, args, logout, true);
  }

  if (args["p"] == "glihelp") {
    produce_glihelp_zipfile(disp, args, logout);    
  }
    
  // Define the page content for the GTI core pages (containing the translation input forms)
  define_gti_core_page(disp, args, logout);
}



void gtiaction::define_gti_home_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  disp.setmacro("gtiformcontent", "gti", "_gti:gtihome_");

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;

  // Get the languages specified in the main.cfg file, and put them into a map to sort by name
  text_tmap gti_languages_name_code_mapping;
  languageinfo_tmap::const_iterator loaded_language = loaded_languages.begin();
  while (loaded_language != loaded_languages.end()) {
    // English is not a valid GTI target language, since it is the source language
    if (loaded_language->first != "en") {
      gti_languages_name_code_mapping[loaded_language->second.longname] = loaded_language->first;
    }
    ++loaded_language;
  }

  // Set the gtitlcselection macro
  text_t gti_tlc_selection = "<select name=\"tlc\">\n";
  text_tmap::iterator gti_language = gti_languages_name_code_mapping.begin();
  while (gti_language != gti_languages_name_code_mapping.end()) {
    gti_tlc_selection += "<option value=\"" + gti_language->second + "\">" + gti_language->first + "</option>\n";
    ++gti_language;
  }
  gti_tlc_selection += "</select>";
  disp.setmacro("gtitlcselection", "gti", gti_tlc_selection);
}



void gtiaction::define_gti_lang_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  // Get the target language code from the CGI arguments
  text_t target_language_code = args["tlc"];

  disp.setmacro("gtiformcontent", "gti", "_gti:gtilang_");

  // Send a request to gti.pl to get the valid translation files
  text_t gti_arguments = "get-language-status " + target_language_code;
  GTI_Response gti_response = parse_gti_response(do_gti_request(gti_arguments, logout), logout);
  if (gti_response.error_message != "") {
    // An error has occurred
    disp.setmacro("gtiformcontent", "gti", "_gti:gtierror_");
    disp.setmacro("gtierrormessage", "gti", gti_response.error_message);
    return;
  }

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;
  disp.setmacro("gtitargetlanguagename", "gti", loaded_languages[target_language_code].longname);

  // Set the gtitfkselection macro
  text_t gti_tfk_selection = "<table>";
  text_tmap::iterator translation_file = gti_response.translation_files_index_to_key_mapping.begin();
  while (translation_file != gti_response.translation_files_index_to_key_mapping.end()) {
    text_t translation_file_key = translation_file->second;

    gti_tfk_selection += "<tr><td>";
    gti_tfk_selection += "<input type=\"radio\" name=\"tfk\" value=\"" + translation_file_key + "\"></td>\n";
    gti_tfk_selection += "<td>_textgti" + translation_file_key + "_</td></tr>\n";

    text_t num_chunks_translated = gti_response.translation_files_key_to_num_chunks_translated_mapping[translation_file_key];
    text_t num_chunks_requiring_translation = gti_response.translation_files_key_to_num_chunks_requiring_translation_mapping[translation_file_key];
    text_t num_chunks_requiring_updating = gti_response.translation_files_key_to_num_chunks_requiring_updating_mapping[translation_file_key];
    gti_tfk_selection += "<tr><td></td><td>_gtitranslationfilestatus_(" + num_chunks_translated + "," + num_chunks_requiring_translation + "," + num_chunks_requiring_updating + ")</td></tr>\n";
    ++translation_file;
  }
  gti_tfk_selection += "</table>";
  disp.setmacro("gtitfkselection", "gti", gti_tfk_selection);
}



void gtiaction::define_gti_status_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  disp.setmacro("gtiformcontent", "gti", "_gti:gtistatus_");

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;

  // Get the languages specified in the main.cfg file, and put them into a map to sort by name
  text_tmap gti_languages_name_code_mapping;
  languageinfo_tmap::const_iterator loaded_language = loaded_languages.begin();
  while (loaded_language != loaded_languages.end()) {
    // English is not a valid GTI target language, since it is the source language
    if (loaded_language->first != "en") {
      gti_languages_name_code_mapping[loaded_language->second.longname] = loaded_language->first;
    }
    ++loaded_language;
  }

  // Get the languages, for each language, send a request to gti.pl to get the valid translation files and the current status for each file
  text_t gti_status_table = "<table class=\"status\">\n";
  text_tmap::iterator gti_language = gti_languages_name_code_mapping.begin();
  bool first_lang = true;
  while (gti_language != gti_languages_name_code_mapping.end()) {    
    // Send a request to gti.pl to get the valid translation files
    text_t gti_arguments = "get-language-status " + gti_language->second;
    GTI_Response gti_response = parse_gti_response(do_gti_request(gti_arguments, logout), logout);
    if (gti_response.error_message != "") {
      // An error has occurred
      disp.setmacro("gtiformcontent", "gti", "_gti:gtierror_");
      disp.setmacro("gtierrormessage", "gti", gti_response.error_message);
      return;
    }

    text_tmap::iterator translation_file = gti_response.translation_files_index_to_key_mapping.begin();
        
    text_t lang_status_temp = "<tr><td class=\"first\">" + gti_language->first + "</td>\n";
    text_t files_temp = "<tr><th class=\"status\">_textgtilanguage_</th>\n";
    text_t number_of_strings_temp = "<tr><td class=\"first\"><b>_textgtitotalnumberoftranslations_</b></td>\n";
    
    while (translation_file != gti_response.translation_files_index_to_key_mapping.end()) {
      text_t translation_file_key = translation_file->second;                     

      text_t num_chunks_translated = gti_response.translation_files_key_to_num_chunks_translated_mapping[translation_file_key];
      text_t num_chunks_requiring_translation = gti_response.translation_files_key_to_num_chunks_requiring_translation_mapping[translation_file_key];
        text_t num_chunks_requiring_updating = gti_response.translation_files_key_to_num_chunks_requiring_updating_mapping[translation_file_key];           
        
        lang_status_temp += "<td class=\"status\">";
        if(num_chunks_translated.getint() > 0){
            lang_status_temp += "<div class=\"nowrap\"><div class=\"done\">";
            lang_status_temp += num_chunks_translated+"</div><div class=\"plus\">+</div><div class=\"update\">";
        lang_status_temp += num_chunks_requiring_updating+"</div><div class=\"plus\">+</div><div class=\"todo\">";
        lang_status_temp += num_chunks_requiring_translation+"</div></div>";
        }
        lang_status_temp += "</td>\n";
        
        
    //lang_status_temp += "<td valign=\"top\" nowrap>_gtitranslationfilestatus2_(" + num_chunks_translated + "," + num_chunks_requiring_translation + "," + num_chunks_requiring_updating + ")</td>";           
        
    // List the file names as the first row of the status table 
    // Add up number of strings need to be translate in each file, as the second line of the status table
      if (first_lang) {
        files_temp += "<th class=\"status\">_textgti" + translation_file_key + "_</th>\n";
        int int_number_of_strings = num_chunks_translated.getint() + num_chunks_requiring_translation.getint();
        number_of_strings_temp += "<td class=\"status\"><b>";
        number_of_strings_temp.appendint(int_number_of_strings);
        number_of_strings_temp += "</b></td>\n";
      }
        ++translation_file;
    }    
    
    if(first_lang) {     
     gti_status_table += files_temp + "</tr>" + number_of_strings_temp + "</tr>";   
     first_lang = false;
    }
    
    gti_status_table += lang_status_temp + "</tr>";    
    ++gti_language;
  }
  gti_status_table += "\n</table>";
  disp.setmacro("gtistatustable", "gti", gti_status_table);
}



void gtiaction::define_gti_find_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  // Get the target language code and file to translate from the CGI arguments
  text_t target_language_code = args["tlc"];
  text_t translation_file_key = args["tfk"];
  text_t query_string = to_utf8(args["q"]);

  // Process user corrections
  if (args["sp"] != "") {
    process_gti_submissions(disp, args, logout, false);
  }

  disp.setmacro("gtiformcontent", "gti", "_gti:gtifind_");

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;
  disp.setmacro("gtitargetlanguagename", "gti", loaded_languages[target_language_code].longname);
  disp.setmacro("gtitranslationfiledesc", "gti", "_gti:textgti" + translation_file_key + "_");
  disp.setmacro("gtitranslationfiledescHtmlsafe", "gti", "_gti:textgti" + encodeForHTML(translation_file_key) + "_");

  if (query_string == "") {
    // No query, so no search results
    disp.setmacro("gtifindformcontent", "gti", "");
    return;
  }

  // Display text right to left if target language is Arabic or Farsi or Urdu
  if (target_language_code == "ar" || target_language_code == "fa" || target_language_code == "ur") {
    disp.setmacro("gtitextdirection", "gti", "rtl");
  }
  else {
    disp.setmacro("gtitextdirection", "gti", "ltr");
  }

  // Send a request to gti.pl to get the valid translation files
  logout << "Query argument: " << query_string << endl;
  text_t gti_arguments = "search-chunks " + target_language_code + " " + translation_file_key + " " + query_string;
  GTI_Response gti_response = parse_gti_response(do_gti_request(gti_arguments, logout), logout);
  if (gti_response.error_message != "") {
    // An error has occurred
    disp.setmacro("gtiformcontent", "gti", "_gti:gtierror_");
    disp.setmacro("gtierrormessage", "gti", gti_response.error_message);
    return;
  }

  disp.setmacro("gtinumchunksmatchingquery", "gti", gti_response.num_chunks_matching_query);

  // Loop through the chunks returned, displaying them on the page
  text_t gti_find_form_content = "_gtifindformheader_\n";
  text_tmap::iterator chunk_key_iterator = gti_response.target_file_chunks_key_to_text_mapping.begin();
  while (chunk_key_iterator != gti_response.target_file_chunks_key_to_text_mapping.end()) {
    text_t chunk_key = chunk_key_iterator->first;

    // Need to escape any underscores in the chunk key to show it correctly on the page
    text_t chunk_key_escaped = escape_all(chunk_key, '_');

    // Need to escape any backslashes, underscores, commas, parentheses, and single quotes in the chunk text
    text_t target_file_chunk_text = gti_response.target_file_chunks_key_to_text_mapping[chunk_key];
    text_t target_file_chunk_text_escaped = escape_all(target_file_chunk_text, '\\');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '_');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, ',');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '(');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, ')');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '"');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '\'');

    // This chunk matches the query
    gti_find_form_content += "_gtichunkmatchingquery_(" + chunk_key_escaped + "," + target_file_chunk_text_escaped + ")\n";

    chunk_key_iterator++;
  }
  gti_find_form_content += "_gtifindformfooter_\n";

  disp.setmacro("gtifindformcontent", "gti", gti_find_form_content);
}



void gtiaction::define_gti_offline_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  // Get the target language code and file to translate from the CGI arguments
  text_t target_language_code = args["tlc"];
  text_t translation_file_key = args["tfk"];

  disp.setmacro("gtiformcontent", "gti", "_gti:gtioffline_");

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;
  disp.setmacro("gtitargetlanguagename", "gti", loaded_languages[target_language_code].longname);
  disp.setmacro("gtitranslationfiledesc", "gti", "_gti:textgti" + translation_file_key + "_");
  disp.setmacro("gtitranslationfiledescHtmlsafe", "gti", "_gti:textgti" + encodeForHTML(translation_file_key) + "_");
}



void gtiaction::define_gti_core_page(displayclass& disp, cgiargsclass& args, ostream& logout)
{
  // Get the target language code and file to translate from the CGI arguments
  text_t target_language_code = args["tlc"];
  text_t translation_file_key = args["tfk"];
  text_t num_chunks_per_page = args["ncpp"];
  logout << "Num chunks per page: " << num_chunks_per_page << endl;

  disp.setmacro("gtiformcontent", "gti", "_gti:gticore_");

  // Display text right to left if target language is Arabic or Farsi or Urdu
  if (target_language_code == "ar" || target_language_code == "fa" || target_language_code == "ur") {
    disp.setmacro("gtitextdirection", "gti", "rtl");
  }
  else {
    disp.setmacro("gtitextdirection", "gti", "ltr");
  }

  // Send a request to gti.pl to get the first string to translate
  text_t gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + num_chunks_per_page;
  GTI_Response gti_response = parse_gti_response(do_gti_request(gti_arguments, logout), logout);
  if (gti_response.error_message != "") {
    // An error has occurred
    disp.setmacro("gtiformcontent", "gti", "_gti:gtierror_");
    disp.setmacro("gtierrormessage", "gti", gti_response.error_message);
    return;
  }

  languageinfo_tmap loaded_languages = recpt->get_configinfo().languages;
  disp.setmacro("gtitargetlanguagename", "gti", loaded_languages[target_language_code].longname);
  if (translation_file_key == "glihelp") {
    disp.setmacro("gtitargetfilepath", "gti", "_gtidownloadglihelp_");
  } else {
    disp.setmacro("gtitargetfilepath", "gti", gti_response.translation_files_key_to_target_file_path_mapping[translation_file_key]);
  }
  disp.setmacro("gtitranslationfiledesc", "gti", "_gti:textgti" + translation_file_key + "_");
  disp.setmacro("gtitranslationfiledescHtmlsafe", "gti", "_gti:textgti" + encodeForHTML(translation_file_key) + "_");
  disp.setmacro("gtiviewtranslationfileinaction", "gti", "_gti:gtiview" + translation_file_key + "inaction_");
  disp.setmacro("gtiviewtranslationfileinactionHtmlsafe", "gti", "_gti:gtiview" + encodeForHTML(translation_file_key) + "inaction_");

  disp.setmacro("gtinumchunkstranslated", "gti", gti_response.translation_files_key_to_num_chunks_translated_mapping[translation_file_key]);
  disp.setmacro("gtinumchunksrequiringtranslation", "gti", gti_response.translation_files_key_to_num_chunks_requiring_translation_mapping[translation_file_key]);
  disp.setmacro("gtinumchunksrequiringupdating", "gti", gti_response.translation_files_key_to_num_chunks_requiring_updating_mapping[translation_file_key]);

  // Check if the translation file is finished
  if (gti_response.translation_files_key_to_num_chunks_requiring_translation_mapping[translation_file_key] == "0" && gti_response.translation_files_key_to_num_chunks_requiring_updating_mapping[translation_file_key] == "0") {
    disp.setmacro("gtiformcontent", "gti", "_gti:gtidone_");
    return;
  }

  // Loop through the chunks returned, displaying them on the page
  text_t gti_core_form_content = "";
  text_tmap::iterator chunk_key_iterator = gti_response.source_file_chunks_key_to_text_mapping.begin();
  while (chunk_key_iterator != gti_response.source_file_chunks_key_to_text_mapping.end()) {
    text_t chunk_key = chunk_key_iterator->first;

    // Need to escape any underscores in the chunk key to show it correctly on the page
    text_t chunk_key_escaped = escape_all(chunk_key, '_');

    // Need to escape any backslashes, underscores, commas, parentheses, and single quotes in the chunk text
    text_t source_file_chunk_text = gti_response.source_file_chunks_key_to_text_mapping[chunk_key];
    text_t source_file_chunk_text_escaped = escape_all(source_file_chunk_text, '\\');
    source_file_chunk_text_escaped = escape_all(source_file_chunk_text_escaped, '_');
    source_file_chunk_text_escaped = escape_all(source_file_chunk_text_escaped, ',');
    source_file_chunk_text_escaped = escape_all(source_file_chunk_text_escaped, '(');
    source_file_chunk_text_escaped = escape_all(source_file_chunk_text_escaped, ')');
    source_file_chunk_text_escaped = escape_all(source_file_chunk_text_escaped, '\'');
    text_t target_file_chunk_text = gti_response.target_file_chunks_key_to_text_mapping[chunk_key];
    text_t target_file_chunk_text_escaped = escape_all(target_file_chunk_text, '\\');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '_');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, ',');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '(');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, ')');
    target_file_chunk_text_escaped = escape_all(target_file_chunk_text_escaped, '\'');

    text_t source_file_chunk_date = gti_response.source_file_chunks_key_to_date_mapping[chunk_key];
    text_t target_file_chunk_date = gti_response.target_file_chunks_key_to_date_mapping[chunk_key];

    // This chunk requires translation
    if (target_file_chunk_text == "") {
      gti_core_form_content += "_gtichunkrequiringtranslation_(" + chunk_key_escaped + "," + source_file_chunk_text_escaped + "," + source_file_chunk_date + ")\n";
    }
    // This chunk requires updating
    else {
      gti_core_form_content += "_gtichunkrequiringupdating_(" + chunk_key_escaped + "," + source_file_chunk_text_escaped + "," + source_file_chunk_date + "," + target_file_chunk_text_escaped + "," + target_file_chunk_date + ")\n";
    }

    chunk_key_iterator++;
  }

  disp.setmacro("gticoreformcontent", "gti", gti_core_form_content);
}



void gtiaction::process_gti_submissions(displayclass& disp, cgiargsclass& args, ostream& logout, bool force_submission)
{
  // Get the target language code and file to translate from the CGI arguments
  text_t target_language_code = args["tlc"];
  text_t translation_file_key = args["tfk"];
  text_t submitter_username = args["un"];

  // Submitted chunk arguments contain the language code followed by "::"
  char* source_chunk_key_start_cstr = ((text_t) "en" + "%3A%3A").getcstr();
  char* target_chunk_key_start_cstr = (target_language_code + "%3A%3A").getcstr();

  // Find the cgi arguments with submitted chunk information
  text_t submission_text;
  cgiargsclass::const_iterator cgi_argument = args.begin();
  while (cgi_argument != args.end()) {
    char* cgi_argument_name_cstr = cgi_argument->first.getcstr();

    // Source file text
    if (strncmp(cgi_argument_name_cstr, source_chunk_key_start_cstr, strlen(source_chunk_key_start_cstr)) == 0) {
      submission_text += "<SourceFileText key=\"";
      text_t source_key = &cgi_argument_name_cstr[strlen(source_chunk_key_start_cstr)];
      decode_cgi_arg(source_key);
      submission_text += source_key;
      //submission_text += &cgi_argument_name_cstr[strlen(source_chunk_key_start_cstr)];
      submission_text += "\">\n";

      text_t source_value = xml_safe(decode_commas(args[cgi_argument->first]));
      // if (args["w"] != "utf-8") {
      // source_value = to_utf8(source_value);
      // }
      submission_text += source_value + "\n";
      submission_text += "</SourceFileText>\n";
    }
    // Target file text
    if (strncmp(cgi_argument_name_cstr, target_chunk_key_start_cstr, strlen(target_chunk_key_start_cstr)) == 0) {
      submission_text += "<TargetFileText key=\"";
      text_t target_key = &cgi_argument_name_cstr[strlen(target_chunk_key_start_cstr)];
      decode_cgi_arg(target_key);
      submission_text += target_key;
      //submission_text += &cgi_argument_name_cstr[strlen(target_chunk_key_start_cstr)];
      submission_text += "\">\n";

      text_t target_value = xml_safe(decode_commas(args[cgi_argument->first]));
      // if (args["w"] != "utf-8") {
      // target_value = to_utf8(target_value);
      // }
      submission_text += target_value + "\n";
      submission_text += "</TargetFileText>\n";
    }

    delete[] cgi_argument_name_cstr;
    ++cgi_argument;
  }

  logout << "Submission text: " << submission_text << endl;

  // Send the submission to gti.pl
  text_t gti_arguments = "submit-translations " + target_language_code + " " + translation_file_key + " " + submitter_username;
  if (force_submission) {
    gti_arguments += " -force_submission";
  }
  do_gti_submission(gti_arguments, submission_text, logout);
  logout << "Done." << endl;

  delete[] source_chunk_key_start_cstr;
  delete[] target_chunk_key_start_cstr;
}



bool gtiaction::produce_excel_spreadsheet(cgiargsclass& args, ostream& logout)
{
  // Get the target language code and file to translate from the CGI arguments
  text_t target_language_code = args["tlc"];
  text_t translation_file_key = args["tfk"];
  text_t target_chunk_type = args["tct"];

  // Send a request to gti.pl to get the Excel spreadsheet data
  text_t gti_arguments = "";
  if (target_chunk_type == "work") {

    // works, but /home/nzdl/gti/gti-generate-excel-xml.xsl doesn't entity escape & into &amp;

    //gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";

    // don't actually need to add toplevel gsdlhome to classpath in the following:
    
    gti_arguments = "get-first-n-chunks-requiring-work " + target_language_code + " " + translation_file_key + " " + "10000" + " | " + java + " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";

  } else {
    // works, but /home/nzdl/gti/gti-generate-excel-xml.xsl doesn't entity escape & into &amp;

    //gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | /opt/jdk1.6.0/bin/java -cp /home/nzdl/gti:/home/nzdl/gti/xalan.jar ApplyXSLT /home/nzdl/gti/gti-generate-excel-xml.xsl -";

    // don't actually need to add toplevel gsdlhome to classpath in the following:
    gti_arguments = "get-all-chunks " + target_language_code + " " + translation_file_key + " | " + java +  " -cp " + gsdlhome + path_separator + gsdlhome + "/bin/java/ApplyXSLT.jar org.nzdl.gsdl.ApplyXSLT -c -t " + gsdlhome + "/bin/script/gti-generate-excel-xml.xsl";
  }

  text_t gti_response_xml_text = do_gti_request(gti_arguments, logout);  
  if (gti_response_xml_text == "") {
    // An error has occurred
    return false;
  }

    // Write the Excel spreadsheet data to the browser
  char* gti_response_xml_text_cstr = gti_response_xml_text.getcstr();
  // use fwrite instead of printf as there are %s in the strings
  fwrite(gti_response_xml_text_cstr, 1, strlen(gti_response_xml_text_cstr), stdout);
  delete[] gti_response_xml_text_cstr;

  return true;
}

bool gtiaction::produce_glihelp_zipfile(displayclass& disp, cgiargsclass& args, ostream& logout)
{
 text_t target_language_code = args["tlc"];
 text_t gti_arguments = "create-glihelp-zip-file " + target_language_code;

 do_gti_request(gti_arguments, logout); 

 disp.setmacro("gtiglihelpzipfilepath", "gti", "tmp/" + target_language_code + "_GLIHelp.zip");
 disp.setmacro("gtiglihelpzipfilepathUrlsafe", "gti", "tmp/" + encodeForURL(target_language_code) + "_GLIHelp.zip");

 return true;
}


text_t gtiaction::escape_all(text_t text_string, char character_to_escape)
{
  text_t text_string_escaped = "";

  text_t::iterator text_string_character = text_string.begin();
  while (text_string_character != text_string.end()) {
    if (*text_string_character == character_to_escape) {
      text_string_escaped += "\\";
    }
    text_string_escaped.push_back(*text_string_character);
    text_string_character++;
  }

  return text_string_escaped;
}



char* xml_get_attribute(const char** attributes, char* attribute_name)
{
  for (int i = 0; (attributes[i] != NULL); i += 2) {
    if (strcmp(attribute_name, attributes[i]) == 0) {
      return strdup(attributes[i+1]);
    }
  }

  return NULL;
}


static void XMLCALL gti_response_xml_start_element(void* user_data, const char* element_name_cstr, const char** attributes)
{
  GTI_Response* gti_response = (GTI_Response*) user_data;
  text_t element_name = element_name_cstr;
  cerr << "In startElement() for " << element_name << endl;

  if (element_name == "GTIError") {
    gti_response->recorded_text = "";
    gti_response->is_recording_text = true;
  }

  if (element_name == "TranslationFile") {
    int translation_file_index = gti_response->translation_files_index_to_key_mapping.size();
    gti_response->translation_file_key = xml_get_attribute(attributes, "key");
    gti_response->translation_files_index_to_key_mapping[translation_file_index] = gti_response->translation_file_key;
    gti_response->translation_files_key_to_target_file_path_mapping[gti_response->translation_file_key] = xml_get_attribute(attributes, "target_file_path");
    gti_response->translation_files_key_to_num_chunks_translated_mapping[gti_response->translation_file_key] = xml_get_attribute(attributes, "num_chunks_translated");
    gti_response->translation_files_key_to_num_chunks_requiring_translation_mapping[gti_response->translation_file_key] = xml_get_attribute(attributes, "num_chunks_requiring_translation");
    gti_response->translation_files_key_to_num_chunks_requiring_updating_mapping[gti_response->translation_file_key] = xml_get_attribute(attributes, "num_chunks_requiring_updating");
  }

  if (element_name == "ChunksMatchingQuery") {
    gti_response->num_chunks_matching_query = xml_get_attribute(attributes, "size");
  }

  if (element_name == "Chunk") {
    gti_response->chunk_key = xml_get_attribute(attributes, "key");
  }

  if (element_name == "SourceFileText") {
    gti_response->source_file_chunks_key_to_date_mapping[gti_response->chunk_key] = xml_get_attribute(attributes, "date");
    gti_response->recorded_text = "";
    gti_response->is_recording_text = true;
  }

  if (element_name == "TargetFileText") {
    if (xml_get_attribute(attributes, "date") != NULL) {
      gti_response->target_file_chunks_key_to_date_mapping[gti_response->chunk_key] = xml_get_attribute(attributes, "date");
    }
    gti_response->recorded_text = "";
    gti_response->is_recording_text = true;
  }
}


static void XMLCALL gti_response_xml_end_element(void* user_data, const char* element_name_cstr)
{
  GTI_Response* gti_response = (GTI_Response*) user_data;
  text_t element_name = element_name_cstr;

  if (element_name == "GTIError") {
    gti_response->error_message = to_uni(gti_response->recorded_text);
    gti_response->is_recording_text = false;
  }

  if (element_name == "SourceFileText") {
    gti_response->source_file_chunks_key_to_text_mapping[gti_response->chunk_key] = to_uni(gti_response->recorded_text);
    gti_response->is_recording_text = false;
  }

  if (element_name == "TargetFileText") {
    gti_response->target_file_chunks_key_to_text_mapping[gti_response->chunk_key] = to_uni(gti_response->recorded_text);
    gti_response->is_recording_text = false;
  }
}


static void XMLCALL gti_response_xml_character_data(void *user_data, const char* text_cstr, int text_length)
{
  GTI_Response* gti_response = (GTI_Response*) user_data;
  if (gti_response->is_recording_text) {
    gti_response->recorded_text.appendcarr(text_cstr, text_length);
  }
}



text_t gtiaction::do_gti_request(text_t gti_arguments, ostream& logout)
{
  // Send the request to gti.pl and read the XML output
  text_t gti_command = "perl -S " + filename_cat(gsdlhome, "bin", "script", "gti.pl") + " " + gti_arguments;
  char* gti_command_cstr = gti_command.getcstr();
  FILE *gti_pipe = popen(gti_command_cstr, "r");
  delete[] gti_command_cstr;
  if (gti_pipe == NULL) {
    logout << "Error: Could not open pipe for GTI command " << gti_command << endl;
    return "";
  }

  // Read the gti.pl response
  text_t gti_response_xml_text;
  while (!feof(gti_pipe)) {
    char buffer[1024];
    gti_response_xml_text.appendcarr(buffer, fread(buffer, 1, 1024, gti_pipe));
  }
  pclose(gti_pipe);

  return gti_response_xml_text;
}



GTI_Response gtiaction::parse_gti_response(text_t gti_response_xml_text, ostream& logout)
{
  GTI_Response gti_response;
  gti_response.is_recording_text = false;

  // Parse the gti.pl response (XML)
  logout << "Parsing GTI command response: " << gti_response_xml_text << endl;
  XML_Parser xml_parser = XML_ParserCreate(NULL);
  XML_SetUserData(xml_parser, &gti_response);
  XML_SetElementHandler(xml_parser, gti_response_xml_start_element, gti_response_xml_end_element);
  XML_SetCharacterDataHandler(xml_parser, gti_response_xml_character_data);

  char* gti_response_xml_text_cstr = gti_response_xml_text.getcstr();
  int parse_status = XML_Parse(xml_parser, gti_response_xml_text_cstr, strlen(gti_response_xml_text_cstr), XML_TRUE);
  delete[] gti_response_xml_text_cstr;
  if (parse_status == XML_STATUS_ERROR) {
    logout << "Parse error " << XML_ErrorString(XML_GetErrorCode(xml_parser)) << " at line " << XML_GetCurrentLineNumber(xml_parser) << endl;
    return gti_response;
  }

  XML_ParserFree(xml_parser);

  logout << "Finished parse." << endl;
  return gti_response;
}



void gtiaction::do_gti_submission(text_t gti_arguments, text_t gti_submission, ostream& logout)
{
  // Send the submission to gti.pl
  text_t gti_command = "perl -S " + filename_cat(gsdlhome, "bin", "script", "gti.pl") + " " + gti_arguments;
  char* gti_command_cstr = gti_command.getcstr();
  FILE *gti_pipe = popen(gti_command_cstr, "w");
  delete[] gti_command_cstr;
  if (gti_pipe == NULL) {
    logout << "Error: Could not open pipe for GTI command " << gti_command << endl;
    return;
  }

  // Write the gti.pl submission
  char* gti_submission_cstr = gti_submission.getcstr();
  fwrite(gti_submission_cstr, 1, strlen(gti_submission_cstr), gti_pipe);
  delete[] gti_submission_cstr;

  pclose(gti_pipe);
}



#endif  // GSDL_USE_GTI_ACTION