root/main/trunk/greenstone2/bin/script/gti.pl @ 30735

Revision 30735, 95.7 KB (checked in by ak19, 3 years ago)

Forgot to clean up comments that were no longer necessary

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en";  # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48    'file_type' => "macrofile",
49    'source_file' => "macros/english.dm",
50    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53    'file_type' => "macrofile",
54    'source_file' => "macros/english2.dm",
55    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58#   'file_type' => "macrofile",
59#   'source_file' => "macros/paperspast-english.dm",
60#   'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64    'file_type' => "resource_bundle",
65    'source_file' => "gli/classes/dictionary.properties",
66    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70    'file_type' => "greenstone_xml",
71    'source_file' => "gli/help/en/help.xml",
72    'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76    'file_type' => "resource_bundle",
77    'source_file' => "perllib/strings.properties",
78    'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82    'file_type' => "resource_bundle",
83    'source_file' => "gsinstaller/LanguagePack.properties",
84    'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94    'file_type' => "resource_bundle",
95    'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96    'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102        'file_type' => "resource_bundle",
103        'source_file' => "greenstone3",
104        'target_file' => "greenstone3"
105},
106
107# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
108# from http://svn.greenstone.org/main/trunk/gs3-collection-configs
109{ 'key' => "gs3colcfg",
110    'file_type' => "resource_bundle",
111    'source_file' => "gs3-collection-configs",
112    'target_file' => "gs3-collection-configs"
113}
114];
115
116my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo");
117
118my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
119#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
120
121# Auxilliary GS3 interface files. This list is not used at present
122# Combine with above list if generating translation spreadsheet for all interface files
123my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
124
125my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
126
127# Not: i18n, log4j
128
129sub main
130{
131    # Get the command to process, and any arguments
132    my $gti_command = shift(@_);
133    my @gti_command_arguments = @_;
134    my $module = $_[1];
135
136    # for GS3, set gsdl_root_dir to GSDL3HOME
137    #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
138    #if($ENV{'GSDL3SRCHOME'}) {
139     #   $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
140     #   $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
141    #}
142    #}
143   
144    # Open the GTI log file for appending, or write to STDERR if that fails
145    if (!open(GTI_LOG, ">>$gti_log_file")) {
146        open(GTI_LOG, ">&STDERR");
147    }
148   
149    # Log the command that launched this script
150    &log_message("Command: $0 @ARGV");
151   
152    # Check that a command was supplied
153    if (!$gti_command) {
154        &throw_fatal_error("Missing command.");
155    }     
156   
157    # Process the command
158    if ($gti_command =~ /^get-all-chunks$/i) {
159        # Check that GS3 interface is the target
160        if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg
161            print &get_all_chunks_for_gs3(@gti_command_arguments);
162        } else {
163            print &get_all_chunks(@gti_command_arguments);
164        }
165    }
166    elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
167        if ($module =~ m/^gs3/) {     
168            print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
169        } else {
170            print &get_first_n_chunks_requiring_work(@gti_command_arguments);
171        }
172    }
173    elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
174        if ($module =~ m/^gs3/) {     
175            print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
176        } else {
177            print &get_uptodate_chunks(@gti_command_arguments);
178        }
179    }
180    elsif ($gti_command =~ /^get-language-status$/i) {
181        print &get_language_status(@gti_command_arguments);       
182    }
183    elsif ($gti_command =~ /^search-chunks$/i) {
184        print &search_chunks(@gti_command_arguments);
185    }
186    elsif ($gti_command =~ /^submit-translations$/i) {
187        # This command cannot produce any output since it reads input
188        &submit_translations(@gti_command_arguments);
189    }
190    elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
191        # This command cannot produce any output since it reads input
192        &create_glihelp_zip_file(@gti_command_arguments);
193    }
194    else {
195        # The command was not recognized
196        &throw_fatal_error("Unknown command \"$gti_command\".");
197    }
198}
199
200
201sub throw_fatal_error
202{
203    my $error_message = shift(@_);
204   
205    # Write an XML error response
206    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
207    print "<GTIResponse>\n";
208    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
209    print "</GTIResponse>\n";
210   
211    # Log the error message, then die
212    &log_message("Error: $error_message");
213    die "\n";
214}
215
216
217sub log_message
218{
219    my $log_message = shift(@_);
220    print GTI_LOG time() . " -- " . $log_message . "\n";
221}
222
223
224sub get_all_chunks
225{
226    # The code of the target language (ensure it is lowercase)
227    my $target_language_code = lc(shift(@_));
228    # The key of the file to translate (ensure it is lowercase)
229    my $translation_file_key = lc(shift(@_));
230   
231    # Check that the necessary arguments were supplied
232    if (!$target_language_code || !$translation_file_key) {
233        &throw_fatal_error("Missing command argument.");
234    }
235   
236    # Get (and check) the translation configuration
237    my ($source_file, $target_file, $translation_file_type)
238    = &get_translation_configuration($target_language_code, $translation_file_key);
239   
240    # Parse the source language and target language files
241    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
242    my @source_file_lines = &read_file_lines($source_file_path);
243    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
244   
245    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
246    my @target_file_lines = &read_file_lines($target_file_path);
247    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
248   
249    # Filter out any automatically translated chunks
250    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
251        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
252            delete $source_file_key_to_line_mapping{$chunk_key};
253            delete $target_file_key_to_line_mapping{$chunk_key};
254        }
255    }
256   
257    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
258    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
259    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
260    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
261   
262    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
263    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
264   
265    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
266   
267    return $xml_response;
268}
269
270
271sub get_uptodate_chunks
272{
273    # The code of the target language (ensure it is lowercase)
274    my $target_language_code = lc(shift(@_));
275    # The key of the file to translate (ensure it is lowercase)
276    my $translation_file_key = lc(shift(@_));
277   
278    # Check that the necessary arguments were supplied
279    if (!$target_language_code || !$translation_file_key) {
280        &throw_fatal_error("Missing command argument.");
281    }
282   
283    # Get (and check) the translation configuration
284    my ($source_file, $target_file, $translation_file_type)
285    = &get_translation_configuration($target_language_code, $translation_file_key);
286   
287    # Parse the source language and target language files
288    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
289    my @source_file_lines = &read_file_lines($source_file_path);
290    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
291   
292    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
293    my @target_file_lines = &read_file_lines($target_file_path);
294    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
295   
296    # Filter out any automatically translated chunks
297    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
298        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
299            delete $source_file_key_to_line_mapping{$chunk_key};
300            delete $target_file_key_to_line_mapping{$chunk_key};
301        }
302    }
303   
304    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
305    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
306    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
307    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
308   
309    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
310    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
311 
312
313    # Chunks needing updating are those in the target file that have been more recently edited in the source file
314    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
315    my @uptodate_target_file_keys = ();
316    foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
317        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
318        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
319       
320        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
321       
322        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
323            # &log_message("Chunk with key $chunk_key needs updating.");
324            push(@uptodate_target_file_keys, $chunk_key);
325        }
326    }
327
328    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
329
330    return $xml_response;
331}
332
333
334sub get_first_n_chunks_requiring_work
335{
336    # The code of the target language (ensure it is lowercase)
337    my $target_language_code = lc(shift(@_));
338    # The key of the file to translate (ensure it is lowercase)
339    my $translation_file_key = lc(shift(@_));
340    # The number of chunks to return (defaults to one if not specified)
341    my $num_chunks_to_return = shift(@_) || "1";
342   
343    # Check that the necessary arguments were supplied
344    if (!$target_language_code || !$translation_file_key) {
345        &throw_fatal_error("Missing command argument.");
346    }
347   
348    # Get (and check) the translation configuration
349    my ($source_file, $target_file, $translation_file_type)
350    = &get_translation_configuration($target_language_code, $translation_file_key);
351
352    # Parse the source language and target language files
353    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
354    my @source_file_lines = &read_file_lines($source_file_path);
355    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
356   
357    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
358    my @target_file_lines = &read_file_lines($target_file_path);
359    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
360   
361    # Filter out any automatically translated chunks
362    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
363        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
364            delete $source_file_key_to_line_mapping{$chunk_key};
365            delete $target_file_key_to_line_mapping{$chunk_key};
366        }
367    }
368   
369    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
372    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
373   
374    # Determine the target file chunks requiring translation
375    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
376    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
377   
378    # Determine the target file chunks requiring updating
379    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
380    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
381    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
382    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
383   
384    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
385   
386    return $xml_response;
387}
388
389
390sub get_language_status
391{
392    # The code of the target language (ensure it is lowercase)
393    my $target_language_code = lc(shift(@_));
394   
395    # Check that the necessary arguments were supplied
396    if (!$target_language_code) {
397        &throw_fatal_error("Missing command argument.");
398    }
399   
400    # Form an XML response to the command
401    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
402    $xml_response .= "<GTIResponse>\n";
403    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
404   
405    foreach my $translation_file (@$gti_translation_files) {   
406        my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
407        my $target_file_name = "";
408       
409        if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg
410            my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
411            &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
412           
413            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
414            my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
415           
416            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
417            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
418            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
419            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
420        }
421        else {
422            # Get (and check) the translation configuration
423            my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
424            $target_file_name = $target_file;
425           
426            # Parse the source language and target language files
427            my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
428            my @source_file_lines = &read_file_lines($source_file_path);
429            my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
430           
431            my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
432            my @target_file_lines = &read_file_lines($target_file_path);
433            my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
434           
435            # Filter out any automatically translated chunks
436            foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
437                if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
438                    delete $source_file_key_to_line_mapping{$chunk_key};
439                    delete $target_file_key_to_line_mapping{$chunk_key};
440                }
441            }
442           
443            my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
444            my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
445           
446            # Determine the target file chunks requiring translation
447            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
448           
449            # Determine the target file chunks requiring updating
450            my @target_file_keys_requiring_updating = ();
451            if (-e $target_file_path) {
452                my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
453                my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
454                @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
455            }
456           
457            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
458            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
459            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
460            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
461        }
462       
463        &log_message("Status of " . $translation_file->{'key'});
464        &log_message("Number of source chunks: " . $num_source_chunks);
465        &log_message("Number of target chunks: " . $num_target_chunks);
466        &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
467        &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
468       
469        $xml_response .= "    <TranslationFile"
470        . " key=\"" . $translation_file->{'key'} . "\""
471        . " target_file_path=\"" . $target_file_name . "\""
472        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
473        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
474        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
475    }
476   
477    $xml_response .= "  </LanguageStatus>\n";
478   
479    $xml_response .= "</GTIResponse>\n";
480    return $xml_response;
481}
482
483
484sub search_chunks
485{
486    # The code of the target language (ensure it is lowercase)
487    my $target_language_code = lc(shift(@_));
488    # The key of the file to translate (ensure it is lowercase)
489    my $translation_file_key = lc(shift(@_));
490    # The query string
491    my $query_string = join(' ', @_);
492   
493    # Check that the necessary arguments were supplied
494    if (!$target_language_code || !$translation_file_key || !$query_string) {
495        &throw_fatal_error("Missing command argument.");
496    }
497   
498    my ($source_file, $target_file, $translation_file_type) = ();
499    my %source_file_key_to_text_mapping = ();
500    my %target_file_key_to_text_mapping = ();
501   
502   
503    if ($translation_file_key !~ m/^gs3/) {
504        # Get (and check) the translation configuration
505        ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
506       
507        # Parse the source language and target language files
508        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
509        my @source_file_lines = &read_file_lines($source_file_path);
510        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
511       
512        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
513        my @target_file_lines = &read_file_lines($target_file_path);
514        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
515       
516        # Filter out any automatically translated chunks
517        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
518            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
519                delete $source_file_key_to_line_mapping{$chunk_key};
520                delete $target_file_key_to_line_mapping{$chunk_key};
521            }
522        }
523       
524        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
525        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
526    }
527    else {
528        # Not needed in this case
529        my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
530        &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
531        \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
532    }
533   
534    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
535    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
536   
537    # Determine the target file chunks matching the query
538    my @target_file_keys_matching_query = ();
539    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
540        my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
541        if ($target_file_text =~ /$query_string/i) {
542            # &log_message("Chunk with key $chunk_key matches query.");
543            push(@target_file_keys_matching_query, $chunk_key);
544        }
545    }
546   
547    # Form an XML response to the command
548    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
549    $xml_response .= "<GTIResponse>\n";
550   
551    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
552    foreach my $chunk_key (@target_file_keys_matching_query) {
553        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
554       
555        $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
556        $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
557        $xml_response .= "    </Chunk>\n";
558    }
559    $xml_response .= "  </ChunksMatchingQuery>\n";
560   
561    $xml_response .= "</GTIResponse>\n";
562    return $xml_response;
563}
564
565
566sub submit_translations
567{
568    # The code of the target language (ensure it is lowercase)
569    my $target_language_code = lc(shift(@_));
570    # The key of the file to translate (ensure it is lowercase)
571    my $translation_file_key = lc(shift(@_));
572    # The username of the translation submitter
573    my $submitter_username = shift(@_);
574    # Whether to submit a target chunk even if it hasn't changed
575    my $force_submission_flag = shift(@_);
576   
577    # Check that the necessary arguments were supplied
578    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
579        &log_message("Fatal error (but cannot be thrown): Missing command argument.");
580        die "\n";
581    }
582   
583    my %source_file_key_to_text_mapping = ();
584    my %source_file_key_to_gti_comment_mapping = ();
585    my %target_file_key_to_text_mapping = ();
586    my %target_file_key_to_gti_comment_mapping = ();
587   
588    my (@source_file_lines, @target_file_lines) = ();
589    my ($source_file, $target_file, $translation_file_type);
590   
591   
592    if ($translation_file_key !~ m/^gs3/) {
593        # Get (and check) the translation configuration
594        ($source_file, $target_file, $translation_file_type)
595        = &get_translation_configuration($target_language_code, $translation_file_key);
596       
597        # Parse the source language and target language files
598        @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
599        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
600        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
601        %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
602       
603        @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
604        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
605        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
606        %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
607    }
608    else {
609        &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
610        \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
611    }
612    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
613    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
614   
615    # Submission date
616    my $day = (localtime)[3];
617    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
618    my $year = (localtime)[5] + 1900;
619    my $submission_date = "$day-$month-$year";
620   
621    open(SUBMISSION, "-");
622    my @submission_lines = <SUBMISSION>;
623    close(SUBMISSION);
624   
625    # Remove any nasty carriage returns
626    # &log_message("Submission:");
627    foreach my $submission_line (@submission_lines) {
628        $submission_line =~ s/\r$//;
629        #&log_message("  $submission_line");
630    }
631   
632    my %source_file_key_to_submission_mapping = ();
633    my %target_file_key_to_submission_mapping = ();
634    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
635        # Read source file part of submission
636        if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
637            my $chunk_key = $1;
638           
639            # Read the source file text
640            my $source_file_chunk_text = "";
641            $i++;
642            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
643                $source_file_chunk_text .= $submission_lines[$i];
644                $i++;
645            }
646            $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
647            $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
648           
649            #&log_message("Source file key: $chunk_key");
650            #&log_message("Source file text: $source_file_chunk_text");
651            $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
652        }
653       
654        # Read target file part of submission
655        if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
656            my $chunk_key = $1;
657           
658            # Read the target file text
659            my $target_file_chunk_text = "";
660            $i++;
661            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
662                $target_file_chunk_text .= $submission_lines[$i];
663                $i++;
664            }
665            $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
666            $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
667           
668            #&log_message("Target file key: $chunk_key");
669            #&log_message("Target file text: $target_file_chunk_text");
670            $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
671        }
672    }
673   
674    # -----------------------------------------
675    #   Validate the translation submissions
676    # -----------------------------------------
677   
678    # Check that the translations are valid
679    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
680
681    # Kathy introduced escaped colons ("\:") into chunk keys in properties files (greenstone3/metadata_names), 
682    # but they're not escaped in the submitted XML versions, nor are they escaped in memory (in the $chunk_key)
683
684        # Make sure the submitted chunk still exists in the source file
685        if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
686            &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
687            delete $source_file_key_to_submission_mapping{$chunk_key};
688            delete $target_file_key_to_submission_mapping{$chunk_key};
689            next;
690        }
691       
692        # Make sure the submitted source chunk matches the source file chunk
693        if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
694        #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
695                    #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
696                #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
697
698            &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
699            &log_message("Submission source: |$source_file_key_to_submission_mapping{$chunk_key}|");
700            &log_message("      Source text: |$source_file_key_to_text_mapping{$chunk_key}|");
701            delete $source_file_key_to_submission_mapping{$chunk_key};
702            delete $target_file_key_to_submission_mapping{$chunk_key};
703            next;
704        }
705    }
706   
707    # Apply the submitted translations
708    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
709        # Only apply the submission if it is a change, unless -force_submission has been specified
710        if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
711            $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
712            $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
713        }
714    }
715   
716    if ($translation_file_key !~ m/^gs3/) {
717        eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
718    } else {
719        eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
720    }
721}
722
723
724sub create_glihelp_zip_file
725{
726    my $target_language_code = shift(@_);
727    my $translation_file_key = "glihelp";
728   
729    &log_message("Creating GLI Help zip file for $target_language_code");
730   
731    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
732   
733    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
734    my $oldclasspath = $classpath;
735    if ( ! -e $classpath) {
736    $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
737    }
738    if ( ! -e $classpath) {
739        &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
740    }
741
742   
743    my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
744    my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
745    my $os = $^O;
746    my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
747    my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
748    $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
749
750    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
751    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
752   
753    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
754    if ( ! -e $gen_many_html_xsl_filepath) {
755        &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
756    }
757   
758    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
759    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
760   
761    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
762    $target_file_directory = $target_file_directory."/";
763   
764    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
765
766    # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
767    my ($tailname, $glihelp_lang_dir, $suffix) =  &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");   
768    if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
769
770    # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
771    my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
772    my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
773    &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
774
775    # In gli/help/<newlang>/help.xml, replace all occurrences of
776    # <Text id="1">This text in en will be removed for new langcode</Text>
777    # with <!-- Missing translation: 1 -->
778    open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
779    my $help_xml_contents;
780    # Read in the entire contents of the file in one hit
781    sysread(FIN, $help_xml_contents, -s FIN);
782    close(FIN);
783   
784    $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
785
786    open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
787    print FOUT $help_xml_contents;
788    close(FOUT);
789    }
790
791    my $perl_exec = &util::get_perl_exec();
792    my $java_exec = "java";
793    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
794        $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
795    }
796
797    #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
798    my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
799    #&throw_fatal_error("RAN gti command: $cmd");
800    my $response = `$cmd`;
801
802    #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
803    $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
804    $response = `$cmd`;
805
806    # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
807    my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
808    if(!&FileUtils::directoryExists($tmpdir)) {
809    &FileUtils::makeDirectory($tmpdir);
810    }
811    #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";   
812    my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
813    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
814
815    $response = `$cmd`;
816}
817
818
819sub get_translation_configuration
820{
821    # Get the code of the target language
822    my $target_language_code = shift(@_);
823    # Get the key of the file to translate
824    my $translation_file_key = shift(@_);
825   
826    # Read the translation data from the gti.cfg file
827    my ($source_file, $target_file, $translation_file_type) =
828    &get_translation_data_for($target_language_code, $translation_file_key);
829   
830    # Check that the file to translate is defined in the gti.cfg file
831    if (!$source_file || !$target_file || !$translation_file_type) {
832        &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
833    }
834   
835    # Check that the source file exists
836    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
837    if (!-e $source_file_path) {
838        &throw_fatal_error("Source file $source_file_path does not exist.");
839    }
840   
841    # Check that the source file is up to date
842    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
843    # unless ($translation_file_is_not_in_cvs) {
844    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
845    my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
846    if ($source_file_cvs_status =~ /^C /) {
847        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
848    }
849    if ($source_file_cvs_status =~ /^M /) {
850        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
851    }
852    # }
853   
854    return ($source_file, $target_file, $translation_file_type);
855}
856
857
858sub get_translation_data_for
859{
860    my ($target_language_code, $translation_file_key) = @_;
861   
862    foreach my $translation_file (@$gti_translation_files) {
863        # If this isn't the correct translation file, move onto the next one
864        next if ($translation_file_key ne $translation_file->{'key'});
865       
866        # Resolve the target language file
867        my $target_language_file = $translation_file->{'target_file'};
868        if ($target_language_file =~ /(\{.+\;.+\})/) {
869            my $unresolved_target_language_file_part = $1;
870           
871            # Check for a special case for the target language code
872            if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
873                my $resolved_target_language_file_part = $2;
874                $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
875            }
876            # Otherwise use the last part as the default value
877            else {
878                my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
879            $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
880        }
881    }
882   
883    # Resolve instances of {iso_639_1_target_language_name}
884    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
885    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
886    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
887   
888    # Resolve instances of {target_language_code}
889    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
890   
891    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
892}
893
894return ();
895}
896
897
898sub read_file_lines
899{
900    my ($file_path) = @_;
901   
902    if (!open(FILE_IN, "<$file_path")) {
903        &log_message("Note: Could not open file $file_path.");
904        return ();
905    }
906    my @file_lines = <FILE_IN>;
907    close(FILE_IN);
908   
909    return @file_lines;
910}
911
912
913sub build_key_to_line_mapping
914{
915    my ($file_lines, $translation_file_type) = @_;
916    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
917}
918
919
920sub build_key_to_text_mapping
921{
922    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
923   
924    my %key_to_text_mapping = ();
925    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
926        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
927        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
928       
929        my $chunk_text = @$file_lines[$chunk_starting_line];
930        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
931            $chunk_text .= @$file_lines[$l];
932        }
933       
934        # Map from chunk key to text
935        eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
936
937        #if($chunk_key =~ m/document\\/) {
938            #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
939            #&log_message("@@@ chunk key: $chunk_key");
940        #}
941
942    }
943   
944    return %key_to_text_mapping;
945}
946
947
948sub build_key_to_last_update_date_mapping
949{
950    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
951   
952    # If the files aren't in CVS then we can't tell anything about what needs updating
953    # return () if ($translation_file_is_not_in_cvs);
954   
955    # Build a mapping from key to CVS date
956    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
957    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
958   
959    # Build a mapping from key to comment date
960    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
961   
962    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
963    my %key_to_last_update_date_mapping = ();
964    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
965        # Use the CVS date as a starting point
966        my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
967        $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
968       
969        # If a comment date exists and it is after the CVS date, use that instead
970        # need to convert the comment date format to SVN format
971        my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
972        if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
973            my $chunk_comment_date = $1;           
974            if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
975                $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;         
976            }
977        }
978    }
979   
980    return %key_to_last_update_date_mapping;
981}
982
983
984sub build_key_to_cvs_date_mapping
985{
986    my ($filename, $translation_file_type) = @_;
987   
988    # Use SVN to annotate each line of the file with the date it was last edited
989    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
990    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
991   
992    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
993   
994    my @cvs_annotated_file_lines_date = ();
995    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
996        # Extract the date from the SVN annotation at the front
997        # svn format : 2007-07-16
998        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
999       
1000        push(@cvs_annotated_file_lines_date, $1);
1001       
1002        # trim extra date information in svn annotation format
1003        # 15:42:49 +1200 (Wed, 21 Jun 2006)
1004        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1005    }   
1006   
1007    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1008    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1009   
1010    my %key_to_cvs_date_mapping = ();
1011    foreach my $chunk_key (keys(%key_to_line_mapping)) {
1012        my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1013        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1014       
1015        # Find the date this chunk was last edited, from the CVS annotation
1016        my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
1017        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1018            if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1019                # This part of the chunk has been updated more recently
1020                $chunk_date = $cvs_annotated_file_lines_date[$l];
1021               
1022            }
1023        }
1024       
1025        # Map from chunk key to CVS date
1026        $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1027    }
1028   
1029    return %key_to_cvs_date_mapping;
1030}
1031
1032
1033sub build_key_to_gti_comment_mapping
1034{
1035    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1036   
1037    my %key_to_gti_comment_mapping = ();
1038    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1039        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1040        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1041       
1042        my $chunk_text = @$file_lines[$chunk_starting_line];
1043        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1044            $chunk_text .= @$file_lines[$l];
1045        }
1046       
1047        # Map from chunk key to GTI comment
1048        my $chunk_gti_comment;
1049        eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1050        $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1051    }
1052   
1053    return %key_to_gti_comment_mapping;
1054}
1055
1056
1057sub determine_chunks_requiring_translation
1058{
1059    my $source_file_key_to_text_mapping = shift(@_);
1060    my $target_file_key_to_text_mapping = shift(@_);
1061   
1062    # Chunks needing translation are those in the source file with no translation in the target file
1063    my @target_file_keys_requiring_translation = ();
1064    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1065        if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1066            # &log_message("Chunk with key $chunk_key needs translating.");
1067            push(@target_file_keys_requiring_translation, $chunk_key);
1068        }
1069    }
1070   
1071    return @target_file_keys_requiring_translation;
1072}
1073
1074
1075sub determine_chunks_requiring_updating
1076{
1077    my $source_file_key_to_last_update_date_mapping = shift(@_);
1078    my $target_file_key_to_last_update_date_mapping = shift(@_);
1079   
1080    # Chunks needing updating are those in the target file that have been more recently edited in the source file
1081    my @target_file_keys_requiring_updating = ();
1082    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1083        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1084        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1085       
1086        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1087
1088        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1089            # &log_message("Chunk with key $chunk_key needs updating.");
1090                # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1091            push(@target_file_keys_requiring_updating, $chunk_key);
1092        }
1093    }
1094   
1095    return @target_file_keys_requiring_updating;
1096}
1097
1098
1099sub is_chunk_automatically_translated
1100{
1101    my ($chunk_key, $translation_file_type) = @_;
1102    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1103}
1104
1105
1106sub make_text_xml_safe
1107{
1108    my $text = shift(@_);
1109    $text =~ s/\&/\&amp\;/g;
1110    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1111    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1112    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1113    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1114    $text =~ s/</\&lt\;/g;
1115    $text =~ s/>/\&gt\;/g;
1116    return $text;
1117}
1118
1119
1120sub unmake_text_xml_safe
1121{
1122    my $text = shift(@_);
1123    $text =~ s/\&lt\;/</g;
1124    $text =~ s/\&gt\;/>/g;
1125    $text =~ s/\&amp\;/\&/g;
1126    return $text;
1127}
1128
1129
1130# Returns 1 if $date1 is after $date2, 0 otherwise
1131sub is_date_after_cvs
1132{
1133    my ($date1, $date2) = @_;
1134    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
1135    "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1136   
1137    if(!defined $date1) {
1138        return 1;
1139    }
1140   
1141    my @date1parts = split(/-/, $date1);
1142    my @date2parts = split(/-/, $date2);
1143   
1144    # Compare year - nasty because we have rolled over into a new century
1145    my $year1 = $date1parts[2];
1146    if ($year1 < 80) {
1147        $year1 += 2000;
1148    }
1149    my $year2 = $date2parts[2];
1150    if ($year2 < 80) {
1151        $year2 += 2000;
1152    }
1153   
1154    # Compare year
1155    if ($year1 > $year2) {
1156        return 1;
1157    }
1158    elsif ($year1 == $year2) {
1159        # Year is the same, so compare month
1160        if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1161            return 1;
1162        }
1163        elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1164            # Month is the same, so compare day
1165            if ($date1parts[0] > $date2parts[0]) {
1166                return 1;
1167            }
1168        }
1169    }
1170   
1171    return 0;
1172}
1173
1174sub is_date_after
1175{
1176    my ($date1, $date2) = @_;
1177   
1178    if(!defined $date1) {
1179        return 1;
1180    }
1181    if(!defined $date2) {
1182        return 0;
1183    }
1184   
1185    # 16-Aug-2006
1186    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1187        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1188        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1189        $date1=$3 . "-" . $months{$2} . "-" . $1;
1190        # print "** converted date1: $date1\n";
1191    }
1192    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1193        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1194        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1195        $date2=$3 . "-" . $months{$2} . "-" . $1;
1196        # print "** converted date2: $date2\n";
1197    }
1198   
1199   
1200    # 2006-08-16
1201    my @date1parts = split(/-/, $date1);
1202    my @date2parts = split(/-/, $date2);
1203   
1204    # Compare year
1205    if ($date1parts[0] > $date2parts[0]) {
1206        return 1;
1207    }
1208    elsif ($date1parts[0] == $date2parts[0]) {
1209        # Year is the same, so compare month
1210        if ($date1parts[1] > $date2parts[1]) {
1211            return 1;
1212        }
1213        elsif ($date1parts[1] == $date2parts[1]) {
1214            # Month is the same, so compare day
1215            if ($date1parts[2] > $date2parts[2]) {
1216                return 1;
1217            }
1218        }
1219    }   
1220   
1221    return 0;
1222}
1223
1224
1225sub create_xml_response_for_chunks_requiring_work
1226{
1227    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1228   
1229    # Form an XML response to the command
1230    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1231    $xml_response .= "<GTIResponse>\n";
1232    $xml_response .= "  <TranslationFile"
1233    . " key=\"" . $translation_file_key . "\""
1234    . " target_file_path=\"" . $target_file . "\""
1235    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1236    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1237    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1238   
1239    # Do chunks requiring translation first
1240    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1241        $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1242    }
1243    else {
1244        $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1245    }
1246   
1247    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1248    foreach my $chunk_key (@sorted_chunk_keys) {
1249        last if ($num_chunks_to_return == 0);
1250       
1251        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1252        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1253       
1254        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1255        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1256        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1257        $xml_response .= "    </Chunk>\n";
1258       
1259        $num_chunks_to_return--;
1260    }
1261   
1262    $xml_response .= "  </ChunksRequiringTranslation>\n";
1263   
1264    # Then do chunks requiring updating
1265    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1266        $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1267    }
1268    else {
1269        $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1270    }
1271   
1272    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1273    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1274    foreach my $chunk_key (@sorted_chunk_keys) {
1275        last if ($num_chunks_to_return == 0);
1276       
1277        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1278        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1279        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1280        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1281       
1282        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1283        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1284        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1285        $xml_response .= "    </Chunk>\n";
1286       
1287        $num_chunks_to_return--;
1288    }
1289   
1290    $xml_response .= "  </ChunksRequiringUpdating>\n";
1291   
1292    $xml_response .= "</GTIResponse>\n";
1293   
1294    return $xml_response;
1295}
1296
1297sub create_xml_response_for_uptodate_chunks
1298{
1299    my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1300   
1301    # Form an XML response to the command
1302    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1303    $xml_response .= "<GTIResponse>\n";
1304    $xml_response .= "  <TranslationFile"
1305    . " key=\"" . $translation_file_key . "\""
1306    . " target_file_path=\"" . $target_file . "\""
1307    . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1308   
1309   
1310    # Then do chunks requiring updating
1311    $xml_response .= "  <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1312   
1313   
1314    # foreach my $chunk_key (@uptodate_target_file_keys) {
1315    my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1316    foreach my $chunk_key (@sorted_chunk_keys) {
1317       
1318        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1319        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1320        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1321        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1322       
1323        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1324        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1325        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1326        $xml_response .= "    </Chunk>\n";
1327
1328    }
1329   
1330    $xml_response .= "  </UptodateChunks>\n";
1331   
1332    $xml_response .= "</GTIResponse>\n";
1333   
1334    return $xml_response;
1335}
1336
1337sub create_xml_response_for_all_chunks
1338{
1339    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1340   
1341    # Form an XML response to the command
1342    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1343    $xml_response .= "<GTIResponse>\n";
1344    $xml_response .= "  <TranslationFile"
1345    . " key=\"" . $translation_file_key . "\""
1346    . " target_file_path=\"" . $target_file . "\"\/>\n";
1347   
1348    # Do all the chunks
1349    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1350   
1351    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1352    foreach my $chunk_key (@sorted_chunk_keys) {
1353        my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1354        my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1355       
1356        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1357        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1358        if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1359            my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1360            my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1361            $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1362        }
1363        else {
1364            $xml_response .= "      <TargetFileText></TargetFileText>\n";
1365        }
1366       
1367        $xml_response .= "    </Chunk>\n";
1368    }
1369    $xml_response .= "  </Chunks>\n";
1370   
1371    $xml_response .= "</GTIResponse>\n";
1372    return $xml_response;
1373}
1374
1375
1376
1377# ==========================================================================================
1378#   MACROFILE FUNCTIONS
1379
1380sub build_key_to_line_mapping_for_macrofile
1381{
1382    my (@file_lines) = @_;
1383   
1384    my $macro_package;
1385    my %chunk_key_to_line_mapping = ();
1386    # Process the contents of the file, line by line
1387    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1388        my $line = $file_lines[$i];
1389        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1390       
1391        # Check if a new package is being defined
1392        if ($line =~ m/^package\s+(.+)/) {
1393            $macro_package = $1;
1394        }
1395       
1396        # Line contains a macro name
1397        elsif ($line =~ m/^(_\w+_)/) {
1398            my $macro_key = $1;
1399            $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1400           
1401            # While there is still text of the macro to go...
1402            my $startline = $i;
1403            while ($line !~ /\}$/) {
1404                $i++;
1405                if ($i == scalar(@file_lines)) {
1406                    &throw_fatal_error("Could not find end of macro $macro_key.");
1407                }
1408                $line = $file_lines[$i];
1409                $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1410            }
1411       
1412        # The chunk key consists of the package name and the macro key
1413        my $chunk_key = $macro_package . "." . $macro_key;
1414        # Map from chunk key to line
1415        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1416    }
1417   
1418    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1419    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1420    # The chunk key consists of package name and macro key
1421    my $chunk_key = $macro_package . "." . $1;
1422    # Map from chunk key to line
1423    $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1424}
1425}
1426
1427return %chunk_key_to_line_mapping;
1428}
1429
1430
1431sub import_chunk_from_macrofile
1432{
1433    my ($chunk_text) = @_;
1434   
1435    # Is this an icon macro??
1436    if ($chunk_text =~ /^\#\# (.*)/) {
1437        # Extract image macro text
1438        $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1439        $chunk_text = $1;
1440   
1441    # Remove enclosing quotes
1442    $chunk_text =~ s/^\"//;
1443    $chunk_text =~ s/\"$//;
1444    }
1445
1446    # No, so it must be a text macro
1447    else {
1448    # Remove macro key
1449    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1450   
1451    # Remove language specifier
1452    $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1453   
1454    # Remove braces enclosing text
1455    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1456    }
1457
1458    return $chunk_text;
1459}
1460
1461
1462sub get_macrofile_chunk_gti_comment
1463{
1464    my ($chunk_text) = @_;
1465   
1466    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1467    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1468        return $1;
1469}
1470
1471return undef;
1472}
1473
1474
1475sub is_macrofile_chunk_automatically_translated
1476{
1477    my ($chunk_key) = @_;
1478   
1479    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1480    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1481        return 1;
1482    }
1483   
1484    return 0;
1485}
1486
1487
1488# Use the source file to generate a target file that is formatted the same
1489sub write_translated_macrofile
1490{
1491    my $source_file = shift(@_);  # Not used
1492    my @source_file_lines = @{shift(@_)};
1493    my $source_file_key_to_text_mapping = shift(@_);
1494    my $target_file = shift(@_);
1495    my @target_file_lines = @{shift(@_)};
1496    my $target_file_key_to_text_mapping = shift(@_);
1497    my $target_file_key_to_gti_comment_mapping = shift(@_);
1498    my $target_language_code = shift(@_);
1499   
1500    # Build a mapping from source file line to chunk key
1501    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1502    my %source_file_line_to_key_mapping = ();
1503    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1504        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1505    }
1506    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1507    my $source_file_line_number = 0;
1508   
1509    # Build a mapping from target file line to chunk key
1510    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1511    my %target_file_line_to_key_mapping = ();
1512    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1513        $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1514    }
1515    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1516   
1517    # Write the new target file
1518    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1519    if (!open(TARGET_FILE, ">$target_file_path")) {
1520        &throw_fatal_error("Could not write target file $target_file_path.");
1521    }
1522   
1523    # Use the header from the target file, to keep language and author information
1524    if (scalar(@target_file_line_keys) > 0) {
1525        my $target_file_line_number = 0;
1526        my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1527        while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1528            my $target_file_line = $target_file_lines[$target_file_line_number];
1529            last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1530                print TARGET_FILE $target_file_line;
1531            $target_file_line_number++;
1532        }
1533       
1534        $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1535    }
1536   
1537    # Model the new target file on the source file, with the target file translations
1538    foreach my $line_key (@source_file_line_keys) {
1539        # Fill in the gaps before this chunk starts
1540        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1541        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1542        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1543            print TARGET_FILE $source_file_lines[$source_file_line_number];
1544            $source_file_line_number++;
1545        }
1546        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1547       
1548        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1549        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1550        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1551       
1552        my $macrofile_key = $chunk_key;
1553        $macrofile_key =~ s/^(.+?)\.//;
1554       
1555        # If no translation exists for this chunk, show this, and move on
1556        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1557            print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1558            next;
1559        }
1560       
1561        # Grab the source chunk text
1562        my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1563        for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1564            $source_file_chunk .= $source_file_lines[$l];
1565        }
1566       
1567        # Is this an icon macro??
1568        if ($source_file_chunk =~ /^\#\# (.*)/) {
1569            # Escape any newline and question mark characters so the source text is replaced correctly
1570            $source_file_chunk_text =~ s/\\/\\\\/g;
1571        $source_file_chunk_text =~ s/\?/\\\?/g;
1572       
1573        # Build the new target chunk from the source chunk
1574        my $target_file_chunk = $source_file_chunk;
1575        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1576        $target_file_chunk =~ s/(\s)*$//;
1577        print TARGET_FILE "$target_file_chunk";
1578    }
1579   
1580    # No, it is just a normal text macro
1581    else {
1582        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1583    }
1584   
1585    # Add the "updated" comment, if one exists
1586    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1587        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1588    }
1589    print TARGET_FILE "\n";
1590}
1591
1592close(TARGET_FILE);
1593}
1594
1595
1596sub sort_by_line
1597{
1598    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1599}
1600
1601
1602# ==========================================================================================
1603#   RESOURCE BUNDLE FUNCTIONS
1604
1605# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1606sub build_key_to_line_mapping_for_resource_bundle
1607{
1608    my (@file_lines) = @_;
1609   
1610    my %chunk_key_to_line_mapping = ();
1611
1612    my $chunk_key;
1613    my $startindex = -1;
1614
1615    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1616        my $line = $file_lines[$i];
1617        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1618       
1619        # a property line has a colon/equals sign as separator that is NOT escaped with a backslash (both keys and values
1620        # can use the colon or = sign. But in the key, such a char is always escaped. Unfortunately, they've not always been
1621        # escaped in the values. So we get the left most occurrence by not doing a greedy match (use ? to not be greedy).
1622        # So find the first :/= char not preceded by \. That will be the true separator of a chunk_key and its value chunk_text
1623
1624        if ($line =~ m/^(\S*?[^\\])[:|=](.*)$/) {
1625            # Line contains a dictionary string
1626
1627            # Unused but useful: http://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
1628            # http://perldoc.perl.org/perlvar.html
1629           
1630            $chunk_key = $1;
1631            # remove the escaping of any :/= property separator from the chunk_key in memory,
1632            # to make comparison with its unescaped version during submissions easier. Will write out with escaping.
1633            $chunk_key =~ s/\\([:=])/$1/g;       
1634           
1635            $startindex = $i;
1636        }       
1637        if ($startindex != -1) {
1638            if($line !~ m/\\$/) { # line finished
1639            # $i keeps track of the line at which this property (chunk_key) finishes
1640
1641            # Map from chunk key to line
1642            $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1643            $startindex = -1;
1644            $chunk_key = "";
1645            }
1646        }       
1647    }
1648   
1649    return %chunk_key_to_line_mapping;
1650}
1651
1652
1653sub import_chunk_from_resource_bundle
1654{
1655    my ($chunk_text) = @_;
1656   
1657    # Simple: just remove string key.
1658    # But key can contain an escaped separator (\: or \=).
1659    # So just as in the previous subroutine, find the first (leftmost) : or = char not preceded by \.
1660    # That will be the true separator of a chunk_key and its value chunk_text
1661    $chunk_text =~ s/^(\S*?[^\\])[:|=](\s*)//s;
1662
1663    $chunk_text =~ s/(\s*)$//s;  # Remove any nasty whitespace, carriage returns etc.
1664    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1665   
1666    return $chunk_text;
1667}
1668
1669
1670sub get_resource_bundle_chunk_gti_comment
1671{
1672    my ($chunk_text) = @_;
1673   
1674    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1675    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1676        return $1;
1677    }
1678
1679    return undef;
1680}
1681
1682
1683sub is_resource_bundle_chunk_automatically_translated
1684{
1685    # No resource bundle chunks are automatically translated
1686    return 0;
1687}
1688
1689
1690sub write_translated_resource_bundle
1691{
1692    my $source_file = shift(@_);  # Not used
1693    my @source_file_lines = @{shift(@_)};
1694    my $source_file_key_to_text_mapping = shift(@_);
1695    my $target_file = shift(@_);
1696    my @target_file_lines = @{shift(@_)};  # Not used
1697    my $target_file_key_to_text_mapping = shift(@_);
1698    my $target_file_key_to_gti_comment_mapping = shift(@_);
1699    my $target_language_code = shift(@_);  # Not used
1700   
1701    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1702    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1703    my %source_file_line_to_key_mapping = ();
1704    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1705        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1706    }
1707   
1708    # Write the new target file
1709    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1710    if (!open(TARGET_FILE, ">$target_file_path")) {
1711        &throw_fatal_error("Could not write target file $target_file_path.");
1712    }
1713   
1714    # Model the new target file on the source file, with the target file translations
1715    my $source_file_line_number = 0;
1716    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1717        # Fill in the gaps before this chunk starts
1718        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1719        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1720        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1721            print TARGET_FILE $source_file_lines[$source_file_line_number];
1722            $source_file_line_number++;
1723        }
1724        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1725       
1726        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1727        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1728        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1729       
1730        # make sure any : or = sign in the chunk key is escaped again (with \) when written out
1731        # since the key-value separator in a property resource bundle file is : or =
1732        my $escaped_chunk_key = $chunk_key;
1733        $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
1734       
1735        # If no translation exists for this chunk, show this, and move on
1736        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1737            print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
1738            next;
1739        }
1740
1741        print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
1742        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1743            print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1744        }
1745        print TARGET_FILE "\n";
1746    }
1747   
1748    close(TARGET_FILE);
1749}
1750
1751
1752# ==========================================================================================
1753#   GREENSTONE XML FUNCTIONS
1754
1755sub build_key_to_line_mapping_for_greenstone_xml
1756{
1757    my (@file_lines) = @_;
1758   
1759    my %chunk_key_to_line_mapping = ();
1760    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1761        my $line = $file_lines[$i];
1762        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1763       
1764        # Line contains a string to translate
1765        if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1766            my $chunk_key = $1;
1767            $line =~ s/\s*$//;  # Remove any nasty whitespace
1768            $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1769           
1770            # While there is still text of the string to go...
1771            my $startline = $i;
1772            while ($line !~ /<\/Text>$/) {
1773                $i++;
1774                if ($i == scalar(@file_lines)) {
1775                    &throw_fatal_error("Could not find end of string $chunk_key.");
1776                }
1777                $line = $file_lines[$i];
1778                $line =~ s/\s*$//;  # Remove any nasty whitespace
1779                $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1780            }
1781           
1782            # Map from chunk key to line
1783            if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1784                $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1785            }
1786            else {
1787                &throw_fatal_error("Duplicate key $chunk_key.");
1788            }
1789        }
1790    }
1791   
1792    return %chunk_key_to_line_mapping;
1793}
1794
1795
1796sub import_chunk_from_greenstone_xml
1797{
1798    my ($chunk_text) = @_;
1799   
1800    # Simple: just remove the Text tags
1801    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1802    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1803    $chunk_text =~ s/<\/Text>$//;
1804   
1805    return $chunk_text;
1806}
1807
1808
1809sub get_greenstone_xml_chunk_gti_comment
1810{
1811    my ($chunk_text) = @_;
1812   
1813    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1814    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1815        return $1;
1816    }
1817   
1818    return undef;
1819}
1820
1821
1822sub is_greenstone_xml_chunk_automatically_translated
1823{
1824    # No greenstone XML chunks are automatically translated
1825    return 0;
1826}
1827
1828
1829sub write_translated_greenstone_xml
1830{
1831    my $source_file = shift(@_);  # Not used
1832    my @source_file_lines = @{shift(@_)};
1833    my $source_file_key_to_text_mapping = shift(@_);
1834    my $target_file = shift(@_);
1835    my @target_file_lines = @{shift(@_)};  # Not used
1836    my $target_file_key_to_text_mapping = shift(@_);
1837    my $target_file_key_to_gti_comment_mapping = shift(@_);
1838    my $target_language_code = shift(@_);  # Not used
1839   
1840    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1841    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1842    my %source_file_line_to_key_mapping = ();
1843    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1844        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1845    }
1846   
1847    # Write the new target file
1848    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1849    if (!open(TARGET_FILE, ">$target_file_path")) {
1850        &throw_fatal_error("Could not write target file $target_file_path.");
1851    }
1852   
1853    # Model the new target file on the source file, with the target file translations
1854    my $source_file_line_number = 0;
1855    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1856        # Fill in the gaps before this chunk starts
1857        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1858        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1859        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1860            print TARGET_FILE $source_file_lines[$source_file_line_number];
1861            $source_file_line_number++;
1862        }
1863        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1864       
1865        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1866        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1867        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1868        $target_file_chunk_text =~ s/(\n)*$//g;
1869       
1870        # If no translation exists for this chunk, show this, and move on
1871        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1872            print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1873            next;
1874        }
1875       
1876        print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1877        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1878            my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1879            $chunk_gti_comment =~ s/^Updated //;
1880            print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1881        }
1882        print TARGET_FILE "\n";
1883    }
1884   
1885    # Fill in the end of the file
1886    while ($source_file_line_number < scalar(@source_file_lines)) {
1887        print TARGET_FILE $source_file_lines[$source_file_line_number];
1888        $source_file_line_number++;
1889    }
1890   
1891    close(TARGET_FILE);
1892}
1893
1894
1895# ==========================================================================================
1896#   GREENSTONE3 FUNCTIONS
1897
1898sub get_all_chunks_for_gs3
1899{
1900    # The code of the target language (ensure it is lowercase)
1901    my $target_language_code = lc(shift(@_));
1902    my $translation_file_key = lc(shift(@_));
1903   
1904    # Check that the necessary arguments were supplied
1905    if (!$target_language_code) {
1906        &throw_fatal_error("Missing command argument.");
1907    }
1908   
1909    # Get (and check) the translation configuration
1910    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1911   
1912    my %source_files_key_to_text_mapping = ();
1913    my %target_files_key_to_text_mapping = ();
1914    my %source_files_key_to_last_update_date_mapping = ();
1915    my %target_files_key_to_last_update_date_mapping = ();
1916   
1917    &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1918   
1919    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1920    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1921   
1922    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1923    return $xml_response;
1924}
1925
1926
1927sub get_first_n_chunks_requiring_work_for_gs3
1928{
1929    # The code of the target language (ensure it is lowercase)
1930    my $target_language_code = lc(shift(@_));
1931    # The key of the file to translate (ensure it is lowercase)
1932    my $translation_file_key = lc(shift(@_));
1933    # The number of chunks to return (defaults to one if not specified)
1934    my $num_chunks_to_return = shift(@_) || "1";
1935   
1936    # Check that the necessary arguments were supplied
1937    if (!$target_language_code || !$translation_file_key) {
1938        &throw_fatal_error("Missing command argument.");
1939    }
1940
1941    my %source_files_key_to_text_mapping = ();
1942    my %target_files_key_to_text_mapping = ();
1943    my %source_files_key_to_last_update_date_mapping = ();
1944    my %target_files_key_to_last_update_date_mapping = ();
1945   
1946    &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1947    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1948   
1949    # Determine the target file chunks requiring translation
1950    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1951    # Determine the target file chunks requiring updating
1952    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1953    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1954    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1955
1956    my $download_target_filepath = "";
1957
1958
1959    # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1960
1961    # if there is no copy of the language files for download, there's also no link to the spreadsheet
1962    # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1963    # interface *.properties files,and link to that zip as the file for offline translation.
1964    # Selecting only properties files for English and the language they're working on (if the last exists)
1965
1966    # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1967    # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1968
1969    my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1970    my $tar_cmd = "tar -cvzf $zip";
1971
1972
1973    # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1974    my $curdir = `pwd`;
1975    chdir $gsdl_root_directory;
1976
1977    $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
1978    $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
1979
1980    # tar command will overwrite the previous version, but want to check we've created it
1981    if(&FileUtils::fileExists($zip)) {
1982    &FileUtils::removeFiles($zip);
1983    }
1984
1985    #my $tar_result = system($tar_cmd); # works but then interface breaks
1986    `$tar_cmd`;
1987    my $tar_result = $?;
1988
1989    if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1990    $download_target_filepath = $zip;
1991    } else {
1992    &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1993    }
1994
1995    # change back to original working directory (cgi-bin/linux probably)
1996    chdir $curdir;
1997
1998    # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
1999
2000
2001    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
2002    \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
2003    $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2004    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2005   
2006    return $xml_response;
2007}
2008
2009# helper function
2010# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg)
2011# formatted correctly to go into a zip file
2012sub get_gs3_zip_file_listing
2013{
2014   my $target_language_code = shift(@_);
2015   my $sourcedir = shift(@_);
2016   my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
2017
2018   my $filelisting = "";
2019   foreach my $interface_file (@$files_array) {
2020
2021    my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
2022    my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2023   
2024    $filelisting = "$filelisting $source_filepath";
2025    if(&FileUtils::fileExists($target_filepath)) {
2026        $filelisting = "$filelisting $target_filepath";
2027    }
2028    }
2029
2030   return $filelisting;
2031}
2032
2033sub get_uptodate_chunks_for_gs3
2034{
2035    # The code of the target language (ensure it is lowercase)
2036    my $target_language_code = lc(shift(@_));
2037    # The key of the file to translate (ensure it is lowercase)
2038    my $translation_file_key = lc(shift(@_));
2039    # The number of chunks to return (defaults to one if not specified)
2040    my $num_chunks_to_return = shift(@_) || "1";
2041   
2042    # Check that the necessary arguments were supplied
2043    if (!$target_language_code || !$translation_file_key) {
2044        &throw_fatal_error("Missing command argument.");
2045    }
2046   
2047    my %source_files_key_to_text_mapping = ();
2048    my %target_files_key_to_text_mapping = ();
2049    my %source_files_key_to_last_update_date_mapping = ();
2050    my %target_files_key_to_last_update_date_mapping = ();
2051   
2052    &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2053    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2054   
2055
2056    # Chunks needing updating are those in the target file that have been more recently edited in the source file
2057    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2058    my @uptodate_target_file_keys = ();
2059    foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2060        my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2061        my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2062       
2063        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
2064       
2065        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2066            # &log_message("Chunk with key $chunk_key needs updating.");
2067            push(@uptodate_target_file_keys, $chunk_key);
2068        }
2069    }
2070
2071    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2072   
2073    return $xml_response;
2074}
2075
2076
2077sub build_gs3_configuration
2078{
2079    my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2080    $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2081   
2082    my $source_file_directory = "greenstone3";  # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2083    my $files_array = \@gs3_interface_files;
2084
2085    if($translation_file_key eq "gs3colcfg") {
2086    $source_file_directory = "gs3-collection-configs";
2087    $files_array = \@gs3_col_cfg_files;
2088    }
2089    my $translation_file_type = "resource_bundle";
2090   
2091    foreach my $interface_file_key (@$files_array) {
2092       
2093        &log_message("Greenstone 3 interface file: " . $interface_file_key);
2094       
2095        # Parse the source language and target language files
2096        my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2097        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2098        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2099        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2100        #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
2101       
2102        my %source_file_key_to_gti_comment_or_last_updated_mapping;
2103        if($get_gti_comments_not_last_updated) {
2104            %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
2105        } else {
2106            %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2107        }
2108
2109        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2110        my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2111        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2112        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2113        #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2114       
2115        my %target_file_key_to_gti_comment_or_last_updated_mapping;
2116        if($get_gti_comments_not_last_updated) {
2117            %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2118        } else {
2119            %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2120        }
2121       
2122       
2123        # Filter out any automatically translated chunks
2124        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2125            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2126                delete $source_file_key_to_line_mapping{$chunk_key};
2127                delete $target_file_key_to_line_mapping{$chunk_key};
2128            }
2129        }
2130       
2131        &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2132        &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2133       
2134        foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2135            my $global_chunk_key = "$interface_file_key.$chunk_key";
2136            $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2137            $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2138           
2139            if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2140                $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2141                $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2142            }
2143        }
2144    }
2145}
2146
2147
2148sub write_translated_gs3interface
2149{
2150    my $translation_file_key = shift(@_);
2151    my $source_file_key_to_text_mapping = shift(@_);
2152    my $target_file_key_to_text_mapping = shift(@_);
2153    my $target_file_key_to_gti_comment_mapping = shift(@_);
2154    my $target_language_code = shift(@_);
2155   
2156    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2157   
2158    my %translated_interface_file_keys = ();
2159    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2160        $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2161        if (!defined $translated_interface_file_keys{$1}) {
2162            &log_message("Updated interface file: " . $1); 
2163            $translated_interface_file_keys{$1}="";
2164        }
2165    }
2166    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2167   
2168    my $source_file_directory = "greenstone3";   
2169    $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2170
2171    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2172       
2173        # Build a mapping from chunk key to source file line, and from source file line to chunk key
2174        my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2175        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2176        my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2177        my %source_file_line_to_key_mapping = ();
2178        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2179            $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2180        }
2181       
2182        # Write the new target file
2183        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2184        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2185        if (!open(TARGET_FILE, ">$target_file_path")) {
2186            &throw_fatal_error("Could not write target file $target_file_path.");
2187        }
2188       
2189        # Model the new target file on the source file, with the target file translations
2190        my $source_file_line_number = 0;
2191        foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2192            # Fill in the gaps before this chunk starts
2193            my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2194            my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2195            while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2196                print TARGET_FILE $source_file_lines[$source_file_line_number];
2197                $source_file_line_number++;
2198            }
2199            $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2200           
2201            my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2202            my $global_chunk_key = "$interface_file_key.$chunk_key";
2203            my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2204            my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2205           
2206            # make sure any : or = sign in the chunk key is escaped again (with \) when written out
2207            # since the key-value separator in a property resource bundle file is : or =
2208            my $escaped_chunk_key = $chunk_key;
2209            $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
2210
2211            # If no translation exists for this chunk, show this, and move on
2212            if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2213                print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
2214                next;
2215            }
2216           
2217            print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
2218            if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2219                print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2220            }
2221            print TARGET_FILE "\n";
2222        }
2223       
2224        close(TARGET_FILE);
2225    }           
2226}
2227
2228&main(@ARGV);
Note: See TracBrowser for help on using the browser.