root/main/trunk/greenstone2/bin/script/gti.pl @ 30490

Revision 30490, 90.8 KB (checked in by ak19, 4 years ago)

Kathy has updated the core gs3 interface files.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en";  # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48    'file_type' => "macrofile",
49    'source_file' => "macros/english.dm",
50    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53    'file_type' => "macrofile",
54    'source_file' => "macros/english2.dm",
55    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58#   'file_type' => "macrofile",
59#   'source_file' => "macros/paperspast-english.dm",
60#   'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64    'file_type' => "resource_bundle",
65    'source_file' => "gli/classes/dictionary.properties",
66    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70    'file_type' => "greenstone_xml",
71    'source_file' => "gli/help/en/help.xml",
72    'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76    'file_type' => "resource_bundle",
77    'source_file' => "perllib/strings.properties",
78    'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82    'file_type' => "resource_bundle",
83    'source_file' => "gsinstaller/LanguagePack.properties",
84    'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94    'file_type' => "resource_bundle",
95    'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96    'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102        'file_type' => "resource_bundle",
103        'source_file' => "greenstone3",
104        'target_file' => "greenstone3"
105}
106];
107
108my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
109#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
110
111# Auxilliary GS3 interface files. This list is not used at present
112# Combine with above list if generating translation spreadsheet for all interface files
113my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
114
115my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
116
117# Not: i18n, log4j
118
119sub main
120{
121    # Get the command to process, and any arguments
122    my $gti_command = shift(@_);
123    my @gti_command_arguments = @_;
124    my $module = $_[1];
125
126    # for GS3, set gsdl_root_dir to GSDL3HOME
127    #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
128    #if($ENV{'GSDL3SRCHOME'}) {
129     #   $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
130     #   $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
131    #}
132    #}
133   
134    # Open the GTI log file for appending, or write to STDERR if that fails
135    if (!open(GTI_LOG, ">>$gti_log_file")) {
136        open(GTI_LOG, ">&STDERR");
137    }
138   
139    # Log the command that launched this script
140    &log_message("Command: $0 @ARGV");
141   
142    # Check that a command was supplied
143    if (!$gti_command) {
144        &throw_fatal_error("Missing command.");
145    }     
146   
147    # Process the command
148    if ($gti_command =~ /^get-all-chunks$/i) {
149        # Check that GS3 interface is the target
150        if ($module eq "gs3interface") {
151            print &get_all_chunks_for_gs3(@gti_command_arguments);
152        } else {
153            print &get_all_chunks(@gti_command_arguments);
154        }
155    }
156    elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
157        if ($module eq "gs3interface") {       
158            print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
159        } else {
160            print &get_first_n_chunks_requiring_work(@gti_command_arguments);
161        }
162    }
163    elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
164        if ($module eq "gs3interface") {       
165            print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
166        } else {
167            print &get_uptodate_chunks(@gti_command_arguments);
168        }
169    }
170    elsif ($gti_command =~ /^get-language-status$/i) {
171        print &get_language_status(@gti_command_arguments);       
172    }
173    elsif ($gti_command =~ /^search-chunks$/i) {
174        print &search_chunks(@gti_command_arguments);
175    }
176    elsif ($gti_command =~ /^submit-translations$/i) {
177        # This command cannot produce any output since it reads input
178        &submit_translations(@gti_command_arguments);
179    }
180    elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
181        # This command cannot produce any output since it reads input
182        &create_glihelp_zip_file(@gti_command_arguments);
183    }
184    else {
185        # The command was not recognized
186        &throw_fatal_error("Unknown command \"$gti_command\".");
187    }
188}
189
190
191sub throw_fatal_error
192{
193    my $error_message = shift(@_);
194   
195    # Write an XML error response
196    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
197    print "<GTIResponse>\n";
198    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
199    print "</GTIResponse>\n";
200   
201    # Log the error message, then die
202    &log_message("Error: $error_message");
203    die "\n";
204}
205
206
207sub log_message
208{
209    my $log_message = shift(@_);
210    print GTI_LOG time() . " -- " . $log_message . "\n";
211}
212
213
214sub get_all_chunks
215{
216    # The code of the target language (ensure it is lowercase)
217    my $target_language_code = lc(shift(@_));
218    # The key of the file to translate (ensure it is lowercase)
219    my $translation_file_key = lc(shift(@_));
220   
221    # Check that the necessary arguments were supplied
222    if (!$target_language_code || !$translation_file_key) {
223        &throw_fatal_error("Missing command argument.");
224    }
225   
226    # Get (and check) the translation configuration
227    my ($source_file, $target_file, $translation_file_type)
228    = &get_translation_configuration($target_language_code, $translation_file_key);
229   
230    # Parse the source language and target language files
231    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
232    my @source_file_lines = &read_file_lines($source_file_path);
233    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
234   
235    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
236    my @target_file_lines = &read_file_lines($target_file_path);
237    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
238   
239    # Filter out any automatically translated chunks
240    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
241        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
242            delete $source_file_key_to_line_mapping{$chunk_key};
243            delete $target_file_key_to_line_mapping{$chunk_key};
244        }
245    }
246   
247    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
248    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
249    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
250    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
251   
252    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
253    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
254   
255    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
256   
257    return $xml_response;
258}
259
260
261sub get_uptodate_chunks
262{
263    # The code of the target language (ensure it is lowercase)
264    my $target_language_code = lc(shift(@_));
265    # The key of the file to translate (ensure it is lowercase)
266    my $translation_file_key = lc(shift(@_));
267   
268    # Check that the necessary arguments were supplied
269    if (!$target_language_code || !$translation_file_key) {
270        &throw_fatal_error("Missing command argument.");
271    }
272   
273    # Get (and check) the translation configuration
274    my ($source_file, $target_file, $translation_file_type)
275    = &get_translation_configuration($target_language_code, $translation_file_key);
276   
277    # Parse the source language and target language files
278    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
279    my @source_file_lines = &read_file_lines($source_file_path);
280    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
281   
282    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
283    my @target_file_lines = &read_file_lines($target_file_path);
284    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
285   
286    # Filter out any automatically translated chunks
287    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
288        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
289            delete $source_file_key_to_line_mapping{$chunk_key};
290            delete $target_file_key_to_line_mapping{$chunk_key};
291        }
292    }
293   
294    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
295    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
296    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
297    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
298   
299    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
300    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
301 
302
303    # Chunks needing updating are those in the target file that have been more recently edited in the source file
304    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
305    my @uptodate_target_file_keys = ();
306    foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
307        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
308        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
309       
310        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
311       
312        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
313            # &log_message("Chunk with key $chunk_key needs updating.");
314            push(@uptodate_target_file_keys, $chunk_key);
315        }
316    }
317
318    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
319
320    return $xml_response;
321}
322
323
324sub get_first_n_chunks_requiring_work
325{
326    # The code of the target language (ensure it is lowercase)
327    my $target_language_code = lc(shift(@_));
328    # The key of the file to translate (ensure it is lowercase)
329    my $translation_file_key = lc(shift(@_));
330    # The number of chunks to return (defaults to one if not specified)
331    my $num_chunks_to_return = shift(@_) || "1";
332   
333    # Check that the necessary arguments were supplied
334    if (!$target_language_code || !$translation_file_key) {
335        &throw_fatal_error("Missing command argument.");
336    }
337   
338    # Get (and check) the translation configuration
339    my ($source_file, $target_file, $translation_file_type)
340    = &get_translation_configuration($target_language_code, $translation_file_key);
341
342    # Parse the source language and target language files
343    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
344    my @source_file_lines = &read_file_lines($source_file_path);
345    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
346   
347    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
348    my @target_file_lines = &read_file_lines($target_file_path);
349    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
350   
351    # Filter out any automatically translated chunks
352    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
353        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
354            delete $source_file_key_to_line_mapping{$chunk_key};
355            delete $target_file_key_to_line_mapping{$chunk_key};
356        }
357    }
358   
359    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
360    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
361    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
362    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
363   
364    # Determine the target file chunks requiring translation
365    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
366    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
367   
368    # Determine the target file chunks requiring updating
369    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
372    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
373   
374    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
375   
376    return $xml_response;
377}
378
379
380sub get_language_status
381{
382    # The code of the target language (ensure it is lowercase)
383    my $target_language_code = lc(shift(@_));
384   
385    # Check that the necessary arguments were supplied
386    if (!$target_language_code) {
387        &throw_fatal_error("Missing command argument.");
388    }
389   
390    # Form an XML response to the command
391    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
392    $xml_response .= "<GTIResponse>\n";
393    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
394   
395    foreach my $translation_file (@$gti_translation_files) {   
396        my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
397        my $target_file_name = "";
398       
399        if ($translation_file->{'key'} eq "gs3interface") {
400            my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
401            &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
402           
403            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
404            my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
405           
406            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
407            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
408            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
409            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
410        }
411        else {
412            # Get (and check) the translation configuration
413            my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
414            $target_file_name = $target_file;
415           
416            # Parse the source language and target language files
417            my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
418            my @source_file_lines = &read_file_lines($source_file_path);
419            my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
420           
421            my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
422            my @target_file_lines = &read_file_lines($target_file_path);
423            my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
424           
425            # Filter out any automatically translated chunks
426            foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
427                if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
428                    delete $source_file_key_to_line_mapping{$chunk_key};
429                    delete $target_file_key_to_line_mapping{$chunk_key};
430                }
431            }
432           
433            my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
434            my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
435           
436            # Determine the target file chunks requiring translation
437            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
438           
439            # Determine the target file chunks requiring updating
440            my @target_file_keys_requiring_updating = ();
441            if (-e $target_file_path) {
442                my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
443                my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
444                @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
445            }
446           
447            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
448            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
449            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
450            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
451        }
452       
453        &log_message("Status of " . $translation_file->{'key'});
454        &log_message("Number of source chunks: " . $num_source_chunks);
455        &log_message("Number of target chunks: " . $num_target_chunks);
456        &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
457        &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
458       
459        $xml_response .= "    <TranslationFile"
460        . " key=\"" . $translation_file->{'key'} . "\""
461        . " target_file_path=\"" . $target_file_name . "\""
462        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
463        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
464        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
465    }
466   
467    $xml_response .= "  </LanguageStatus>\n";
468   
469    $xml_response .= "</GTIResponse>\n";
470    return $xml_response;
471}
472
473
474sub search_chunks
475{
476    # The code of the target language (ensure it is lowercase)
477    my $target_language_code = lc(shift(@_));
478    # The key of the file to translate (ensure it is lowercase)
479    my $translation_file_key = lc(shift(@_));
480    # The query string
481    my $query_string = join(' ', @_);
482   
483    # Check that the necessary arguments were supplied
484    if (!$target_language_code || !$translation_file_key || !$query_string) {
485        &throw_fatal_error("Missing command argument.");
486    }
487   
488    my ($source_file, $target_file, $translation_file_type) = ();
489    my %source_file_key_to_text_mapping = ();
490    my %target_file_key_to_text_mapping = ();
491   
492   
493    if ($translation_file_key ne "gs3interface") {
494        # Get (and check) the translation configuration
495        ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
496       
497        # Parse the source language and target language files
498        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
499        my @source_file_lines = &read_file_lines($source_file_path);
500        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
501       
502        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
503        my @target_file_lines = &read_file_lines($target_file_path);
504        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
505       
506        # Filter out any automatically translated chunks
507        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
508            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
509                delete $source_file_key_to_line_mapping{$chunk_key};
510                delete $target_file_key_to_line_mapping{$chunk_key};
511            }
512        }
513       
514        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
515        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
516    }
517    else {
518        # Not needed in this case
519        my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
520        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
521        \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
522    }
523   
524    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
525    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
526   
527    # Determine the target file chunks matching the query
528    my @target_file_keys_matching_query = ();
529    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
530        my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
531        if ($target_file_text =~ /$query_string/i) {
532            # &log_message("Chunk with key $chunk_key matches query.");
533            push(@target_file_keys_matching_query, $chunk_key);
534        }
535    }
536   
537    # Form an XML response to the command
538    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
539    $xml_response .= "<GTIResponse>\n";
540   
541    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
542    foreach my $chunk_key (@target_file_keys_matching_query) {
543        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
544       
545        $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
546        $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
547        $xml_response .= "    </Chunk>\n";
548    }
549    $xml_response .= "  </ChunksMatchingQuery>\n";
550   
551    $xml_response .= "</GTIResponse>\n";
552    return $xml_response;
553}
554
555
556sub submit_translations
557{
558    # The code of the target language (ensure it is lowercase)
559    my $target_language_code = lc(shift(@_));
560    # The key of the file to translate (ensure it is lowercase)
561    my $translation_file_key = lc(shift(@_));
562    # The username of the translation submitter
563    my $submitter_username = shift(@_);
564    # Whether to submit a target chunk even if it hasn't changed
565    my $force_submission_flag = shift(@_);
566   
567    # Check that the necessary arguments were supplied
568    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
569        &log_message("Fatal error (but cannot be thrown): Missing command argument.");
570        die "\n";
571    }
572   
573    my %source_file_key_to_text_mapping = ();
574    my %source_file_key_to_gti_comment_mapping = ();
575    my %target_file_key_to_text_mapping = ();
576    my %target_file_key_to_gti_comment_mapping = ();
577   
578    my (@source_file_lines, @target_file_lines) = ();
579    my ($source_file, $target_file, $translation_file_type);
580   
581   
582    if ($translation_file_key ne "gs3interface") {
583        # Get (and check) the translation configuration
584        ($source_file, $target_file, $translation_file_type)
585        = &get_translation_configuration($target_language_code, $translation_file_key);
586       
587        # Parse the source language and target language files
588        @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
589        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
590        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
591        %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
592       
593        @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
594        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
595        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
596        %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
597    }
598    else {
599        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
600        \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
601    }
602    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
603    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
604   
605    # Submission date
606    my $day = (localtime)[3];
607    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
608    my $year = (localtime)[5] + 1900;
609    my $submission_date = "$day-$month-$year";
610   
611    open(SUBMISSION, "-");
612    my @submission_lines = <SUBMISSION>;
613    close(SUBMISSION);
614   
615    # Remove any nasty carriage returns
616    # &log_message("Submission:");
617    foreach my $submission_line (@submission_lines) {
618        $submission_line =~ s/\r$//;
619        #&log_message("  $submission_line");
620    }
621   
622    my %source_file_key_to_submission_mapping = ();
623    my %target_file_key_to_submission_mapping = ();
624    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
625        # Read source file part of submission
626        if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
627            my $chunk_key = $1;
628           
629            # Read the source file text
630            my $source_file_chunk_text = "";
631            $i++;
632            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
633                $source_file_chunk_text .= $submission_lines[$i];
634                $i++;
635            }
636            $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
637            $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
638           
639            #&log_message("Source file key: $chunk_key");
640            #&log_message("Source file text: $source_file_chunk_text");
641            $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
642        }
643       
644        # Read target file part of submission
645        if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
646            my $chunk_key = $1;
647           
648            # Read the target file text
649            my $target_file_chunk_text = "";
650            $i++;
651            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
652                $target_file_chunk_text .= $submission_lines[$i];
653                $i++;
654            }
655            $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
656            $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
657           
658            #&log_message("Target file key: $chunk_key");
659            #&log_message("Target file text: $target_file_chunk_text");
660            $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
661        }
662    }
663   
664    # -----------------------------------------
665    #   Validate the translation submissions
666    # -----------------------------------------
667   
668    # Check that the translations are valid
669    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
670        # Make sure the submitted chunk still exists in the source file
671        if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
672            &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
673            delete $source_file_key_to_submission_mapping{$chunk_key};
674            delete $target_file_key_to_submission_mapping{$chunk_key};
675            next;
676        }
677       
678        # Make sure the submitted source chunk matches the source file chunk
679        if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
680        #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
681                    #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
682                #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
683
684            &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
685            &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
686            &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
687            delete $source_file_key_to_submission_mapping{$chunk_key};
688            delete $target_file_key_to_submission_mapping{$chunk_key};
689            next;
690        }
691    }
692   
693    # Apply the submitted translations
694    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
695        # Only apply the submission if it is a change, unless -force_submission has been specified
696        if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
697            $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
698            $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
699        }
700    }
701   
702    if ($translation_file_key ne "gs3interface") {
703        eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
704    } else {
705        eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
706    }
707}
708
709
710sub create_glihelp_zip_file
711{
712    my $target_language_code = shift(@_);
713    my $translation_file_key = "glihelp";
714   
715    &log_message("Creating GLI Help zip file for $target_language_code");
716   
717    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
718   
719    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
720    my $oldclasspath = $classpath;
721    if ( ! -e $classpath) {
722    $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
723    }
724    if ( ! -e $classpath) {
725        &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
726    }
727
728   
729    my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
730    my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
731    my $os = $^O;
732    my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
733    my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
734    $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
735
736    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
737    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
738   
739    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
740    if ( ! -e $gen_many_html_xsl_filepath) {
741        &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
742    }
743   
744    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
745    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
746   
747    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
748    $target_file_directory = $target_file_directory."/";
749   
750    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
751
752    # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
753    my ($tailname, $glihelp_lang_dir, $suffix) =  &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");   
754    if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
755
756    # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
757    my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
758    my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
759    &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
760
761    # In gli/help/<newlang>/help.xml, replace all occurrences of
762    # <Text id="1">This text in en will be removed for new langcode</Text>
763    # with <!-- Missing translation: 1 -->
764    open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
765    my $help_xml_contents;
766    # Read in the entire contents of the file in one hit
767    sysread(FIN, $help_xml_contents, -s FIN);
768    close(FIN);
769   
770    $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
771
772    open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
773    print FOUT $help_xml_contents;
774    close(FOUT);
775    }
776
777    my $perl_exec = &util::get_perl_exec();
778    my $java_exec = "java";
779    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
780        $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
781    }
782
783    #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
784    my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
785    #&throw_fatal_error("RAN gti command: $cmd");
786    my $response = `$cmd`;
787
788    #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
789    $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
790    $response = `$cmd`;
791
792    # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
793    my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
794    if(!&FileUtils::directoryExists($tmpdir)) {
795    &FileUtils::makeDirectory($tmpdir);
796    }
797    #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";   
798    my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
799    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
800
801    $response = `$cmd`;
802}
803
804
805sub get_translation_configuration
806{
807    # Get the code of the target language
808    my $target_language_code = shift(@_);
809    # Get the key of the file to translate
810    my $translation_file_key = shift(@_);
811   
812    # Read the translation data from the gti.cfg file
813    my ($source_file, $target_file, $translation_file_type) =
814    &get_translation_data_for($target_language_code, $translation_file_key);
815   
816    # Check that the file to translate is defined in the gti.cfg file
817    if (!$source_file || !$target_file || !$translation_file_type) {
818        &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
819    }
820   
821    # Check that the source file exists
822    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
823    if (!-e $source_file_path) {
824        &throw_fatal_error("Source file $source_file_path does not exist.");
825    }
826   
827    # Check that the source file is up to date
828    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
829    # unless ($translation_file_is_not_in_cvs) {
830    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
831    my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
832    if ($source_file_cvs_status =~ /^C /) {
833        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
834    }
835    if ($source_file_cvs_status =~ /^M /) {
836        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
837    }
838    # }
839   
840    return ($source_file, $target_file, $translation_file_type);
841}
842
843
844sub get_translation_data_for
845{
846    my ($target_language_code, $translation_file_key) = @_;
847   
848    foreach my $translation_file (@$gti_translation_files) {
849        # If this isn't the correct translation file, move onto the next one
850        next if ($translation_file_key ne $translation_file->{'key'});
851       
852        # Resolve the target language file
853        my $target_language_file = $translation_file->{'target_file'};
854        if ($target_language_file =~ /(\{.+\;.+\})/) {
855            my $unresolved_target_language_file_part = $1;
856           
857            # Check for a special case for the target language code
858            if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
859                my $resolved_target_language_file_part = $2;
860                $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
861            }
862            # Otherwise use the last part as the default value
863            else {
864                my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
865            $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
866        }
867    }
868   
869    # Resolve instances of {iso_639_1_target_language_name}
870    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
871    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
872    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
873   
874    # Resolve instances of {target_language_code}
875    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
876   
877    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
878}
879
880return ();
881}
882
883
884sub read_file_lines
885{
886    my ($file_path) = @_;
887   
888    if (!open(FILE_IN, "<$file_path")) {
889        &log_message("Note: Could not open file $file_path.");
890        return ();
891    }
892    my @file_lines = <FILE_IN>;
893    close(FILE_IN);
894   
895    return @file_lines;
896}
897
898
899sub build_key_to_line_mapping
900{
901    my ($file_lines, $translation_file_type) = @_;
902    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
903}
904
905
906sub build_key_to_text_mapping
907{
908    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
909   
910    my %key_to_text_mapping = ();
911    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
912        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
913        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
914       
915        my $chunk_text = @$file_lines[$chunk_starting_line];
916        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
917            $chunk_text .= @$file_lines[$l];
918        }
919       
920        # Map from chunk key to text
921        eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
922    }
923   
924    return %key_to_text_mapping;
925}
926
927
928sub build_key_to_last_update_date_mapping
929{
930    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
931   
932    # If the files aren't in CVS then we can't tell anything about what needs updating
933    # return () if ($translation_file_is_not_in_cvs);
934   
935    # Build a mapping from key to CVS date
936    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
937    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
938   
939    # Build a mapping from key to comment date
940    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
941   
942    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
943    my %key_to_last_update_date_mapping = ();
944    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
945        # Use the CVS date as a starting point
946        my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
947        $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
948       
949        # If a comment date exists and it is after the CVS date, use that instead
950        # need to convert the comment date format to SVN format
951        my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
952        if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
953            my $chunk_comment_date = $1;           
954            if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
955                $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;         
956            }
957        }
958    }
959   
960    return %key_to_last_update_date_mapping;
961}
962
963
964sub build_key_to_cvs_date_mapping
965{
966    my ($filename, $translation_file_type) = @_;
967   
968    # Use SVN to annotate each line of the file with the date it was last edited
969    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
970    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
971   
972    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
973   
974    my @cvs_annotated_file_lines_date = ();
975    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
976        # Extract the date from the SVN annotation at the front
977        # svn format : 2007-07-16
978        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
979       
980        push(@cvs_annotated_file_lines_date, $1);
981       
982        # trim extra date information in svn annotation format
983        # 15:42:49 +1200 (Wed, 21 Jun 2006)
984        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
985    }   
986   
987    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
988    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
989   
990    my %key_to_cvs_date_mapping = ();
991    foreach my $chunk_key (keys(%key_to_line_mapping)) {
992        my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
993        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
994       
995        # Find the date this chunk was last edited, from the CVS annotation
996        my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
997        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
998            if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
999                # This part of the chunk has been updated more recently
1000                $chunk_date = $cvs_annotated_file_lines_date[$l];
1001               
1002            }
1003        }
1004       
1005        # Map from chunk key to CVS date
1006        $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1007    }
1008   
1009    return %key_to_cvs_date_mapping;
1010}
1011
1012
1013sub build_key_to_gti_comment_mapping
1014{
1015    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1016   
1017    my %key_to_gti_comment_mapping = ();
1018    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1019        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1020        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1021       
1022        my $chunk_text = @$file_lines[$chunk_starting_line];
1023        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1024            $chunk_text .= @$file_lines[$l];
1025        }
1026       
1027        # Map from chunk key to GTI comment
1028        my $chunk_gti_comment;
1029        eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1030        $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1031    }
1032   
1033    return %key_to_gti_comment_mapping;
1034}
1035
1036
1037sub determine_chunks_requiring_translation
1038{
1039    my $source_file_key_to_text_mapping = shift(@_);
1040    my $target_file_key_to_text_mapping = shift(@_);
1041   
1042    # Chunks needing translation are those in the source file with no translation in the target file
1043    my @target_file_keys_requiring_translation = ();
1044    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1045        if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1046            # &log_message("Chunk with key $chunk_key needs translating.");
1047            push(@target_file_keys_requiring_translation, $chunk_key);
1048        }
1049    }
1050   
1051    return @target_file_keys_requiring_translation;
1052}
1053
1054
1055sub determine_chunks_requiring_updating
1056{
1057    my $source_file_key_to_last_update_date_mapping = shift(@_);
1058    my $target_file_key_to_last_update_date_mapping = shift(@_);
1059   
1060    # Chunks needing updating are those in the target file that have been more recently edited in the source file
1061    my @target_file_keys_requiring_updating = ();
1062    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1063        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1064        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1065       
1066        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
1067       
1068        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1069            # &log_message("Chunk with key $chunk_key needs updating.");
1070            push(@target_file_keys_requiring_updating, $chunk_key);
1071        }
1072    }
1073   
1074    return @target_file_keys_requiring_updating;
1075}
1076
1077
1078sub is_chunk_automatically_translated
1079{
1080    my ($chunk_key, $translation_file_type) = @_;
1081    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1082}
1083
1084
1085sub make_text_xml_safe
1086{
1087    my $text = shift(@_);
1088    $text =~ s/\&/\&amp\;/g;
1089    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1090    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1091    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1092    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1093    $text =~ s/</\&lt\;/g;
1094    $text =~ s/>/\&gt\;/g;
1095    return $text;
1096}
1097
1098
1099sub unmake_text_xml_safe
1100{
1101    my $text = shift(@_);
1102    $text =~ s/\&lt\;/</g;
1103    $text =~ s/\&gt\;/>/g;
1104    $text =~ s/\&amp\;/\&/g;
1105    return $text;
1106}
1107
1108
1109# Returns 1 if $date1 is after $date2, 0 otherwise
1110sub is_date_after_cvs
1111{
1112    my ($date1, $date2) = @_;
1113    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
1114    "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1115   
1116    if(!defined $date1) {
1117        return 1;
1118    }
1119   
1120    my @date1parts = split(/-/, $date1);
1121    my @date2parts = split(/-/, $date2);
1122   
1123    # Compare year - nasty because we have rolled over into a new century
1124    my $year1 = $date1parts[2];
1125    if ($year1 < 80) {
1126        $year1 += 2000;
1127    }
1128    my $year2 = $date2parts[2];
1129    if ($year2 < 80) {
1130        $year2 += 2000;
1131    }
1132   
1133    # Compare year
1134    if ($year1 > $year2) {
1135        return 1;
1136    }
1137    elsif ($year1 == $year2) {
1138        # Year is the same, so compare month
1139        if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1140            return 1;
1141        }
1142        elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1143            # Month is the same, so compare day
1144            if ($date1parts[0] > $date2parts[0]) {
1145                return 1;
1146            }
1147        }
1148    }
1149   
1150    return 0;
1151}
1152
1153sub is_date_after
1154{
1155    my ($date1, $date2) = @_;
1156   
1157    if(!defined $date1) {
1158        return 1;
1159    }
1160    if(!defined $date2) {
1161        return 0;
1162    }
1163   
1164    # 16-Aug-2006
1165    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1166        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1167        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1168        $date1=$3 . "-" . $months{$2} . "-" . $1;
1169        # print "** converted date1: $date1\n";
1170    }
1171    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1172        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1173        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1174        $date2=$3 . "-" . $months{$2} . "-" . $1;
1175        # print "** converted date2: $date2\n";
1176    }
1177   
1178   
1179    # 2006-08-16
1180    my @date1parts = split(/-/, $date1);
1181    my @date2parts = split(/-/, $date2);
1182   
1183    # Compare year
1184    if ($date1parts[0] > $date2parts[0]) {
1185        return 1;
1186    }
1187    elsif ($date1parts[0] == $date2parts[0]) {
1188        # Year is the same, so compare month
1189        if ($date1parts[1] > $date2parts[1]) {
1190            return 1;
1191        }
1192        elsif ($date1parts[1] == $date2parts[1]) {
1193            # Month is the same, so compare day
1194            if ($date1parts[2] > $date2parts[2]) {
1195                return 1;
1196            }
1197        }
1198    }   
1199   
1200    return 0;
1201}
1202
1203
1204sub create_xml_response_for_chunks_requiring_work
1205{
1206    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1207   
1208    # Form an XML response to the command
1209    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1210    $xml_response .= "<GTIResponse>\n";
1211    $xml_response .= "  <TranslationFile"
1212    . " key=\"" . $translation_file_key . "\""
1213    . " target_file_path=\"" . $target_file . "\""
1214    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1215    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1216    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1217   
1218    # Do chunks requiring translation first
1219    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1220        $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1221    }
1222    else {
1223        $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1224    }
1225   
1226    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1227    foreach my $chunk_key (@sorted_chunk_keys) {
1228        last if ($num_chunks_to_return == 0);
1229       
1230        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1231        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1232       
1233        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1234        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1235        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1236        $xml_response .= "    </Chunk>\n";
1237       
1238        $num_chunks_to_return--;
1239    }
1240   
1241    $xml_response .= "  </ChunksRequiringTranslation>\n";
1242   
1243    # Then do chunks requiring updating
1244    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1245        $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1246    }
1247    else {
1248        $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1249    }
1250   
1251    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1252    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1253    foreach my $chunk_key (@sorted_chunk_keys) {
1254        last if ($num_chunks_to_return == 0);
1255       
1256        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1257        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1258        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1259        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1260       
1261        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1262        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1263        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1264        $xml_response .= "    </Chunk>\n";
1265       
1266        $num_chunks_to_return--;
1267    }
1268   
1269    $xml_response .= "  </ChunksRequiringUpdating>\n";
1270   
1271    $xml_response .= "</GTIResponse>\n";
1272   
1273    return $xml_response;
1274}
1275
1276sub create_xml_response_for_uptodate_chunks
1277{
1278    my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1279   
1280    # Form an XML response to the command
1281    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1282    $xml_response .= "<GTIResponse>\n";
1283    $xml_response .= "  <TranslationFile"
1284    . " key=\"" . $translation_file_key . "\""
1285    . " target_file_path=\"" . $target_file . "\""
1286    . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1287   
1288   
1289    # Then do chunks requiring updating
1290    $xml_response .= "  <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1291   
1292   
1293    # foreach my $chunk_key (@uptodate_target_file_keys) {
1294    my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1295    foreach my $chunk_key (@sorted_chunk_keys) {
1296       
1297        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1298        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1299        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1300        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1301       
1302        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1303        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1304        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1305        $xml_response .= "    </Chunk>\n";
1306
1307    }
1308   
1309    $xml_response .= "  </UptodateChunks>\n";
1310   
1311    $xml_response .= "</GTIResponse>\n";
1312   
1313    return $xml_response;
1314}
1315
1316sub create_xml_response_for_all_chunks
1317{
1318    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1319   
1320    # Form an XML response to the command
1321    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1322    $xml_response .= "<GTIResponse>\n";
1323    $xml_response .= "  <TranslationFile"
1324    . " key=\"" . $translation_file_key . "\""
1325    . " target_file_path=\"" . $target_file . "\"\/>\n";
1326   
1327    # Do all the chunks
1328    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1329   
1330    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1331    foreach my $chunk_key (@sorted_chunk_keys) {
1332        my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1333        my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1334       
1335        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1336        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1337        if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1338            my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1339            my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1340            $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1341        }
1342        else {
1343            $xml_response .= "      <TargetFileText></TargetFileText>\n";
1344        }
1345       
1346        $xml_response .= "    </Chunk>\n";
1347    }
1348    $xml_response .= "  </Chunks>\n";
1349   
1350    $xml_response .= "</GTIResponse>\n";
1351    return $xml_response;
1352}
1353
1354
1355
1356# ==========================================================================================
1357#   MACROFILE FUNCTIONS
1358
1359sub build_key_to_line_mapping_for_macrofile
1360{
1361    my (@file_lines) = @_;
1362   
1363    my $macro_package;
1364    my %chunk_key_to_line_mapping = ();
1365    # Process the contents of the file, line by line
1366    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1367        my $line = $file_lines[$i];
1368        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1369       
1370        # Check if a new package is being defined
1371        if ($line =~ m/^package\s+(.+)/) {
1372            $macro_package = $1;
1373        }
1374       
1375        # Line contains a macro name
1376        elsif ($line =~ m/^(_\w+_)/) {
1377            my $macro_key = $1;
1378            $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1379           
1380            # While there is still text of the macro to go...
1381            my $startline = $i;
1382            while ($line !~ /\}$/) {
1383                $i++;
1384                if ($i == scalar(@file_lines)) {
1385                    &throw_fatal_error("Could not find end of macro $macro_key.");
1386                }
1387                $line = $file_lines[$i];
1388                $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1389            }
1390       
1391        # The chunk key consists of the package name and the macro key
1392        my $chunk_key = $macro_package . "." . $macro_key;
1393        # Map from chunk key to line
1394        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1395    }
1396   
1397    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1398    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1399    # The chunk key consists of package name and macro key
1400    my $chunk_key = $macro_package . "." . $1;
1401    # Map from chunk key to line
1402    $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1403}
1404}
1405
1406return %chunk_key_to_line_mapping;
1407}
1408
1409
1410sub import_chunk_from_macrofile
1411{
1412    my ($chunk_text) = @_;
1413   
1414    # Is this an icon macro??
1415    if ($chunk_text =~ /^\#\# (.*)/) {
1416        # Extract image macro text
1417        $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1418        $chunk_text = $1;
1419   
1420    # Remove enclosing quotes
1421    $chunk_text =~ s/^\"//;
1422    $chunk_text =~ s/\"$//;
1423}
1424
1425# No, so it must be a text macro
1426else {
1427    # Remove macro key
1428    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1429   
1430    # Remove language specifier
1431    $chunk_text =~ s/^\[l=.*\](\s*)//;
1432   
1433    # Remove braces enclosing text
1434    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1435}
1436
1437return $chunk_text;
1438}
1439
1440
1441sub get_macrofile_chunk_gti_comment
1442{
1443    my ($chunk_text) = @_;
1444   
1445    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1446    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1447        return $1;
1448}
1449
1450return undef;
1451}
1452
1453
1454sub is_macrofile_chunk_automatically_translated
1455{
1456    my ($chunk_key) = @_;
1457   
1458    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1459    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1460        return 1;
1461    }
1462   
1463    return 0;
1464}
1465
1466
1467# Use the source file to generate a target file that is formatted the same
1468sub write_translated_macrofile
1469{
1470    my $source_file = shift(@_);  # Not used
1471    my @source_file_lines = @{shift(@_)};
1472    my $source_file_key_to_text_mapping = shift(@_);
1473    my $target_file = shift(@_);
1474    my @target_file_lines = @{shift(@_)};
1475    my $target_file_key_to_text_mapping = shift(@_);
1476    my $target_file_key_to_gti_comment_mapping = shift(@_);
1477    my $target_language_code = shift(@_);
1478   
1479    # Build a mapping from source file line to chunk key
1480    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1481    my %source_file_line_to_key_mapping = ();
1482    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1483        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1484    }
1485    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1486    my $source_file_line_number = 0;
1487   
1488    # Build a mapping from target file line to chunk key
1489    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1490    my %target_file_line_to_key_mapping = ();
1491    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1492        $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1493    }
1494    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1495   
1496    # Write the new target file
1497    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1498    if (!open(TARGET_FILE, ">$target_file_path")) {
1499        &throw_fatal_error("Could not write target file $target_file_path.");
1500    }
1501   
1502    # Use the header from the target file, to keep language and author information
1503    if (scalar(@target_file_line_keys) > 0) {
1504        my $target_file_line_number = 0;
1505        my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1506        while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1507            my $target_file_line = $target_file_lines[$target_file_line_number];
1508            last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1509                print TARGET_FILE $target_file_line;
1510            $target_file_line_number++;
1511        }
1512       
1513        $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1514    }
1515   
1516    # Model the new target file on the source file, with the target file translations
1517    foreach my $line_key (@source_file_line_keys) {
1518        # Fill in the gaps before this chunk starts
1519        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1520        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1521        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1522            print TARGET_FILE $source_file_lines[$source_file_line_number];
1523            $source_file_line_number++;
1524        }
1525        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1526       
1527        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1528        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1529        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1530       
1531        my $macrofile_key = $chunk_key;
1532        $macrofile_key =~ s/^(.+?)\.//;
1533       
1534        # If no translation exists for this chunk, show this, and move on
1535        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1536            print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1537            next;
1538        }
1539       
1540        # Grab the source chunk text
1541        my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1542        for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1543            $source_file_chunk .= $source_file_lines[$l];
1544        }
1545       
1546        # Is this an icon macro??
1547        if ($source_file_chunk =~ /^\#\# (.*)/) {
1548            # Escape any newline and question mark characters so the source text is replaced correctly
1549            $source_file_chunk_text =~ s/\\/\\\\/g;
1550        $source_file_chunk_text =~ s/\?/\\\?/g;
1551       
1552        # Build the new target chunk from the source chunk
1553        my $target_file_chunk = $source_file_chunk;
1554        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1555        $target_file_chunk =~ s/(\s)*$//;
1556        print TARGET_FILE "$target_file_chunk";
1557    }
1558   
1559    # No, it is just a normal text macro
1560    else {
1561        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1562    }
1563   
1564    # Add the "updated" comment, if one exists
1565    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1566        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1567    }
1568    print TARGET_FILE "\n";
1569}
1570
1571close(TARGET_FILE);
1572}
1573
1574
1575sub sort_by_line
1576{
1577    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1578}
1579
1580
1581# ==========================================================================================
1582#   RESOURCE BUNDLE FUNCTIONS
1583
1584# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1585sub build_key_to_line_mapping_for_resource_bundle
1586{
1587    my (@file_lines) = @_;
1588   
1589    my %chunk_key_to_line_mapping = ();
1590
1591    my $chunk_key;
1592    my $startindex = -1;
1593
1594    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1595        my $line = $file_lines[$i];
1596        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1597       
1598        # Line contains a dictionary string
1599        if ($line =~ /^(\S+?)[:|=](.*)$/) {
1600            $chunk_key = $1;
1601            $startindex = $i;
1602        }       
1603        if ($startindex != -1) {
1604            if($line !~ m/\\$/) { # line finished
1605            # $i keeps track of the line at which this property (chunk_key) finishes
1606
1607            # Map from chunk key to line
1608            $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1609            $startindex = -1;
1610            $chunk_key = "";
1611            }
1612        }               
1613    }
1614   
1615    return %chunk_key_to_line_mapping;
1616}
1617
1618
1619sub import_chunk_from_resource_bundle
1620{
1621    my ($chunk_text) = @_;
1622   
1623    # Simple: just remove string key
1624    $chunk_text =~ s/^(\S+?)[:|=](\s*)//s;
1625    $chunk_text =~ s/(\s*)$//s;  # Remove any nasty whitespace, carriage returns etc.
1626    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1627   
1628    return $chunk_text;
1629}
1630
1631
1632sub get_resource_bundle_chunk_gti_comment
1633{
1634    my ($chunk_text) = @_;
1635   
1636    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1637    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1638        return $1;
1639}
1640
1641return undef;
1642}
1643
1644
1645sub is_resource_bundle_chunk_automatically_translated
1646{
1647    # No resource bundle chunks are automatically translated
1648    return 0;
1649}
1650
1651
1652sub write_translated_resource_bundle
1653{
1654    my $source_file = shift(@_);  # Not used
1655    my @source_file_lines = @{shift(@_)};
1656    my $source_file_key_to_text_mapping = shift(@_);
1657    my $target_file = shift(@_);
1658    my @target_file_lines = @{shift(@_)};  # Not used
1659    my $target_file_key_to_text_mapping = shift(@_);
1660    my $target_file_key_to_gti_comment_mapping = shift(@_);
1661    my $target_language_code = shift(@_);  # Not used
1662   
1663    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1664    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1665    my %source_file_line_to_key_mapping = ();
1666    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1667        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1668    }
1669   
1670    # Write the new target file
1671    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1672    if (!open(TARGET_FILE, ">$target_file_path")) {
1673        &throw_fatal_error("Could not write target file $target_file_path.");
1674    }
1675   
1676    # Model the new target file on the source file, with the target file translations
1677    my $source_file_line_number = 0;
1678    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1679        # Fill in the gaps before this chunk starts
1680        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1681        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1682        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1683            print TARGET_FILE $source_file_lines[$source_file_line_number];
1684            $source_file_line_number++;
1685        }
1686        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1687       
1688        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1689        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1690        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1691       
1692        # If no translation exists for this chunk, show this, and move on
1693        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1694            print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1695            next;
1696        }
1697       
1698        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1699        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1700            print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1701        }
1702        print TARGET_FILE "\n";
1703    }
1704   
1705    close(TARGET_FILE);
1706}
1707
1708
1709# ==========================================================================================
1710#   GREENSTONE XML FUNCTIONS
1711
1712sub build_key_to_line_mapping_for_greenstone_xml
1713{
1714    my (@file_lines) = @_;
1715   
1716    my %chunk_key_to_line_mapping = ();
1717    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1718        my $line = $file_lines[$i];
1719        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1720       
1721        # Line contains a string to translate
1722        if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1723            my $chunk_key = $1;
1724            $line =~ s/\s*$//;  # Remove any nasty whitespace
1725            $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1726           
1727            # While there is still text of the string to go...
1728            my $startline = $i;
1729            while ($line !~ /<\/Text>$/) {
1730                $i++;
1731                if ($i == scalar(@file_lines)) {
1732                    &throw_fatal_error("Could not find end of string $chunk_key.");
1733                }
1734                $line = $file_lines[$i];
1735                $line =~ s/\s*$//;  # Remove any nasty whitespace
1736                $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1737            }
1738           
1739            # Map from chunk key to line
1740            if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1741                $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1742            }
1743            else {
1744                &throw_fatal_error("Duplicate key $chunk_key.");
1745            }
1746        }
1747    }
1748   
1749    return %chunk_key_to_line_mapping;
1750}
1751
1752
1753sub import_chunk_from_greenstone_xml
1754{
1755    my ($chunk_text) = @_;
1756   
1757    # Simple: just remove the Text tags
1758    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1759    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1760    $chunk_text =~ s/<\/Text>$//;
1761   
1762    return $chunk_text;
1763}
1764
1765
1766sub get_greenstone_xml_chunk_gti_comment
1767{
1768    my ($chunk_text) = @_;
1769   
1770    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1771    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1772        return $1;
1773    }
1774   
1775    return undef;
1776}
1777
1778
1779sub is_greenstone_xml_chunk_automatically_translated
1780{
1781    # No greenstone XML chunks are automatically translated
1782    return 0;
1783}
1784
1785
1786sub write_translated_greenstone_xml
1787{
1788    my $source_file = shift(@_);  # Not used
1789    my @source_file_lines = @{shift(@_)};
1790    my $source_file_key_to_text_mapping = shift(@_);
1791    my $target_file = shift(@_);
1792    my @target_file_lines = @{shift(@_)};  # Not used
1793    my $target_file_key_to_text_mapping = shift(@_);
1794    my $target_file_key_to_gti_comment_mapping = shift(@_);
1795    my $target_language_code = shift(@_);  # Not used
1796   
1797    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1798    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1799    my %source_file_line_to_key_mapping = ();
1800    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1801        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1802    }
1803   
1804    # Write the new target file
1805    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1806    if (!open(TARGET_FILE, ">$target_file_path")) {
1807        &throw_fatal_error("Could not write target file $target_file_path.");
1808    }
1809   
1810    # Model the new target file on the source file, with the target file translations
1811    my $source_file_line_number = 0;
1812    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1813        # Fill in the gaps before this chunk starts
1814        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1815        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1816        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1817            print TARGET_FILE $source_file_lines[$source_file_line_number];
1818            $source_file_line_number++;
1819        }
1820        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1821       
1822        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1823        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1824        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1825        $target_file_chunk_text =~ s/(\n)*$//g;
1826       
1827        # If no translation exists for this chunk, show this, and move on
1828        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1829            print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1830            next;
1831        }
1832       
1833        print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1834        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1835            my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1836            $chunk_gti_comment =~ s/^Updated //;
1837            print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1838        }
1839        print TARGET_FILE "\n";
1840    }
1841   
1842    # Fill in the end of the file
1843    while ($source_file_line_number < scalar(@source_file_lines)) {
1844        print TARGET_FILE $source_file_lines[$source_file_line_number];
1845        $source_file_line_number++;
1846    }
1847   
1848    close(TARGET_FILE);
1849}
1850
1851
1852# ==========================================================================================
1853#   GREENSTONE3 FUNCTIONS
1854
1855sub get_all_chunks_for_gs3
1856{
1857    # The code of the target language (ensure it is lowercase)
1858    my $target_language_code = lc(shift(@_));
1859    my $translation_file_key = lc(shift(@_));
1860   
1861    # Check that the necessary arguments were supplied
1862    if (!$target_language_code) {
1863        &throw_fatal_error("Missing command argument.");
1864    }
1865   
1866    # Get (and check) the translation configuration
1867    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1868   
1869    my %source_files_key_to_text_mapping = ();
1870    my %target_files_key_to_text_mapping = ();
1871    my %source_files_key_to_last_update_date_mapping = ();
1872    my %target_files_key_to_last_update_date_mapping = ();
1873   
1874    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1875   
1876    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1877    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1878   
1879    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1880    return $xml_response;
1881}
1882
1883
1884sub get_first_n_chunks_requiring_work_for_gs3
1885{
1886    # The code of the target language (ensure it is lowercase)
1887    my $target_language_code = lc(shift(@_));
1888    # The key of the file to translate (ensure it is lowercase)
1889    my $translation_file_key = lc(shift(@_));
1890    # The number of chunks to return (defaults to one if not specified)
1891    my $num_chunks_to_return = shift(@_) || "1";
1892   
1893    # Check that the necessary arguments were supplied
1894    if (!$target_language_code || !$translation_file_key) {
1895        &throw_fatal_error("Missing command argument.");
1896    }
1897
1898    my %source_files_key_to_text_mapping = ();
1899    my %target_files_key_to_text_mapping = ();
1900    my %source_files_key_to_last_update_date_mapping = ();
1901    my %target_files_key_to_last_update_date_mapping = ();
1902   
1903    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1904    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1905   
1906    # Determine the target file chunks requiring translation
1907    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1908    # Determine the target file chunks requiring updating
1909    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1910    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1911    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1912
1913    my $download_target_filepath = "";
1914
1915
1916    # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1917
1918    # if there is no copy of the language files for download, there's also no link to the spreadsheet
1919    # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1920    # interface *.properties files,and link to that zip as the file for offline translation.
1921    # Selecting only properties files for English and the language they're working on (if the last exists)
1922
1923    # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1924    # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1925
1926    my $sourcedir = "greenstone3";
1927    my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1928    my $tar_cmd = "tar -cvzf $zip";
1929
1930
1931    # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1932    my $curdir = `pwd`;
1933    chdir $gsdl_root_directory;
1934
1935    my $filelisting = "";
1936    foreach my $interface_file (@gs3_interface_files) {
1937
1938    my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
1939    my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
1940   
1941    $filelisting = "$filelisting $source_filepath";
1942    if(&FileUtils::fileExists($target_filepath)) {
1943        $filelisting = "$filelisting $target_filepath";
1944    }
1945    }
1946    $tar_cmd .= " $filelisting";
1947
1948    # tar command will overwrite the previous version, but want to check we've created it
1949    if(&FileUtils::fileExists($zip)) {
1950    &FileUtils::removeFiles($zip);
1951    }
1952
1953    #my $tar_result = system($tar_cmd); # works but then interface breaks
1954    `$tar_cmd`;
1955    my $tar_result = $?;
1956
1957    if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1958    $download_target_filepath = $zip;
1959    } else {
1960    &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1961    }
1962
1963    # change back to original working directory (cgi-bin/linux probably)
1964    chdir $curdir;
1965
1966    # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
1967
1968
1969    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
1970    \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1971    $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1972    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1973   
1974    return $xml_response;
1975}
1976
1977sub get_uptodate_chunks_for_gs3
1978{
1979    # The code of the target language (ensure it is lowercase)
1980    my $target_language_code = lc(shift(@_));
1981    # The key of the file to translate (ensure it is lowercase)
1982    my $translation_file_key = lc(shift(@_));
1983    # The number of chunks to return (defaults to one if not specified)
1984    my $num_chunks_to_return = shift(@_) || "1";
1985   
1986    # Check that the necessary arguments were supplied
1987    if (!$target_language_code || !$translation_file_key) {
1988        &throw_fatal_error("Missing command argument.");
1989    }
1990   
1991    my %source_files_key_to_text_mapping = ();
1992    my %target_files_key_to_text_mapping = ();
1993    my %source_files_key_to_last_update_date_mapping = ();
1994    my %target_files_key_to_last_update_date_mapping = ();
1995   
1996    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1997    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1998   
1999
2000    # Chunks needing updating are those in the target file that have been more recently edited in the source file
2001    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2002    my @uptodate_target_file_keys = ();
2003    foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2004        my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2005        my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2006       
2007        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
2008       
2009        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2010            # &log_message("Chunk with key $chunk_key needs updating.");
2011            push(@uptodate_target_file_keys, $chunk_key);
2012        }
2013    }
2014
2015    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2016   
2017    return $xml_response;
2018}
2019
2020
2021sub build_gs3_configuration
2022{
2023    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2024    $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
2025   
2026    my $source_file_directory = "greenstone3";  # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2027    my $translation_file_type = "resource_bundle";
2028   
2029    foreach my $interface_file_key (@gs3_interface_files) {
2030       
2031        &log_message("Greenstone 3 interface file: " . $interface_file_key);
2032       
2033        # Parse the source language and target language files
2034        my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2035        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2036        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2037        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2038        my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2039       
2040        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2041        my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2042        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2043        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2044        my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2045       
2046       
2047        # Filter out any automatically translated chunks
2048        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2049            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2050                delete $source_file_key_to_line_mapping{$chunk_key};
2051                delete $target_file_key_to_line_mapping{$chunk_key};
2052            }
2053        }
2054       
2055        &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2056        &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2057       
2058        foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2059            my $global_chunk_key = "$interface_file_key.$chunk_key";
2060            $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2061            $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
2062           
2063            if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2064                $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2065                $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
2066            }
2067        }
2068    }
2069}
2070
2071
2072sub write_translated_gs3interface
2073{
2074    my $source_file_key_to_text_mapping = shift(@_);
2075    my $target_file_key_to_text_mapping = shift(@_);
2076    my $target_file_key_to_gti_comment_mapping = shift(@_);
2077    my $target_language_code = shift(@_);
2078   
2079    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2080   
2081    my %translated_interface_file_keys = ();
2082    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2083        $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2084        if (!defined $translated_interface_file_keys{$1}) {
2085            &log_message("Updated interface file: " . $1); 
2086            $translated_interface_file_keys{$1}="";
2087        }
2088    }
2089    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2090   
2091    my $source_file_directory = "greenstone3";   
2092   
2093    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2094       
2095        # Build a mapping from chunk key to source file line, and from source file line to chunk key
2096        my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2097        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2098        my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2099        my %source_file_line_to_key_mapping = ();
2100        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2101            $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2102        }
2103       
2104        # Write the new target file
2105        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2106        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2107        if (!open(TARGET_FILE, ">$target_file_path")) {
2108            &throw_fatal_error("Could not write target file $target_file_path.");
2109        }
2110       
2111        # Model the new target file on the source file, with the target file translations
2112        my $source_file_line_number = 0;
2113        foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2114            # Fill in the gaps before this chunk starts
2115            my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2116            my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2117            while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2118                print TARGET_FILE $source_file_lines[$source_file_line_number];
2119                $source_file_line_number++;
2120            }
2121            $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2122           
2123            my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2124            my $global_chunk_key = "$interface_file_key.$chunk_key";
2125            my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2126            my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2127           
2128            # If no translation exists for this chunk, show this, and move on
2129            if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2130                print TARGET_FILE "# -- Missing translation: $chunk_key\n";
2131                next;
2132            }
2133           
2134            print TARGET_FILE "$chunk_key:$target_file_chunk_text";
2135            if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2136                print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2137            }
2138            print TARGET_FILE "\n";
2139        }
2140       
2141        close(TARGET_FILE);
2142    }           
2143}
2144
2145&main(@ARGV);
Note: See TracBrowser for help on using the browser.