root/main/trunk/greenstone2/bin/script/gti.pl @ 30548

Revision 30548, 92.1 KB (checked in by ak19, 4 years ago)

Fixing gti.pl: previously for just the gs3interface module, if any strings that required translating got translated, they'd immediately be added to the list of strings that require updating. This was due to an oversight in build_gs3_configuration() which was not getting the last_updated_mapping when it needed, but always getting the gti_comments instead. Now it gets whichever of these is appropriate, and things work at last.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en";  # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48    'file_type' => "macrofile",
49    'source_file' => "macros/english.dm",
50    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53    'file_type' => "macrofile",
54    'source_file' => "macros/english2.dm",
55    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58#   'file_type' => "macrofile",
59#   'source_file' => "macros/paperspast-english.dm",
60#   'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64    'file_type' => "resource_bundle",
65    'source_file' => "gli/classes/dictionary.properties",
66    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70    'file_type' => "greenstone_xml",
71    'source_file' => "gli/help/en/help.xml",
72    'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76    'file_type' => "resource_bundle",
77    'source_file' => "perllib/strings.properties",
78    'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82    'file_type' => "resource_bundle",
83    'source_file' => "gsinstaller/LanguagePack.properties",
84    'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94    'file_type' => "resource_bundle",
95    'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96    'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102        'file_type' => "resource_bundle",
103        'source_file' => "greenstone3",
104        'target_file' => "greenstone3"
105}
106];
107
108my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
109#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
110
111# Auxilliary GS3 interface files. This list is not used at present
112# Combine with above list if generating translation spreadsheet for all interface files
113my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
114
115my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
116
117# Not: i18n, log4j
118
119sub main
120{
121    # Get the command to process, and any arguments
122    my $gti_command = shift(@_);
123    my @gti_command_arguments = @_;
124    my $module = $_[1];
125
126    # for GS3, set gsdl_root_dir to GSDL3HOME
127    #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
128    #if($ENV{'GSDL3SRCHOME'}) {
129     #   $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
130     #   $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
131    #}
132    #}
133   
134    # Open the GTI log file for appending, or write to STDERR if that fails
135    if (!open(GTI_LOG, ">>$gti_log_file")) {
136        open(GTI_LOG, ">&STDERR");
137    }
138   
139    # Log the command that launched this script
140    &log_message("Command: $0 @ARGV");
141   
142    # Check that a command was supplied
143    if (!$gti_command) {
144        &throw_fatal_error("Missing command.");
145    }     
146   
147    # Process the command
148    if ($gti_command =~ /^get-all-chunks$/i) {
149        # Check that GS3 interface is the target
150        if ($module eq "gs3interface") {
151            print &get_all_chunks_for_gs3(@gti_command_arguments);
152        } else {
153            print &get_all_chunks(@gti_command_arguments);
154        }
155    }
156    elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
157        if ($module eq "gs3interface") {       
158            print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
159        } else {
160            print &get_first_n_chunks_requiring_work(@gti_command_arguments);
161        }
162    }
163    elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
164        if ($module eq "gs3interface") {       
165            print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
166        } else {
167            print &get_uptodate_chunks(@gti_command_arguments);
168        }
169    }
170    elsif ($gti_command =~ /^get-language-status$/i) {
171        print &get_language_status(@gti_command_arguments);       
172    }
173    elsif ($gti_command =~ /^search-chunks$/i) {
174        print &search_chunks(@gti_command_arguments);
175    }
176    elsif ($gti_command =~ /^submit-translations$/i) {
177        # This command cannot produce any output since it reads input
178        &submit_translations(@gti_command_arguments);
179    }
180    elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
181        # This command cannot produce any output since it reads input
182        &create_glihelp_zip_file(@gti_command_arguments);
183    }
184    else {
185        # The command was not recognized
186        &throw_fatal_error("Unknown command \"$gti_command\".");
187    }
188}
189
190
191sub throw_fatal_error
192{
193    my $error_message = shift(@_);
194   
195    # Write an XML error response
196    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
197    print "<GTIResponse>\n";
198    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
199    print "</GTIResponse>\n";
200   
201    # Log the error message, then die
202    &log_message("Error: $error_message");
203    die "\n";
204}
205
206
207sub log_message
208{
209    my $log_message = shift(@_);
210    print GTI_LOG time() . " -- " . $log_message . "\n";
211}
212
213
214sub get_all_chunks
215{
216    # The code of the target language (ensure it is lowercase)
217    my $target_language_code = lc(shift(@_));
218    # The key of the file to translate (ensure it is lowercase)
219    my $translation_file_key = lc(shift(@_));
220   
221    # Check that the necessary arguments were supplied
222    if (!$target_language_code || !$translation_file_key) {
223        &throw_fatal_error("Missing command argument.");
224    }
225   
226    # Get (and check) the translation configuration
227    my ($source_file, $target_file, $translation_file_type)
228    = &get_translation_configuration($target_language_code, $translation_file_key);
229   
230    # Parse the source language and target language files
231    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
232    my @source_file_lines = &read_file_lines($source_file_path);
233    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
234   
235    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
236    my @target_file_lines = &read_file_lines($target_file_path);
237    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
238   
239    # Filter out any automatically translated chunks
240    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
241        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
242            delete $source_file_key_to_line_mapping{$chunk_key};
243            delete $target_file_key_to_line_mapping{$chunk_key};
244        }
245    }
246   
247    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
248    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
249    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
250    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
251   
252    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
253    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
254   
255    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
256   
257    return $xml_response;
258}
259
260
261sub get_uptodate_chunks
262{
263    # The code of the target language (ensure it is lowercase)
264    my $target_language_code = lc(shift(@_));
265    # The key of the file to translate (ensure it is lowercase)
266    my $translation_file_key = lc(shift(@_));
267   
268    # Check that the necessary arguments were supplied
269    if (!$target_language_code || !$translation_file_key) {
270        &throw_fatal_error("Missing command argument.");
271    }
272   
273    # Get (and check) the translation configuration
274    my ($source_file, $target_file, $translation_file_type)
275    = &get_translation_configuration($target_language_code, $translation_file_key);
276   
277    # Parse the source language and target language files
278    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
279    my @source_file_lines = &read_file_lines($source_file_path);
280    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
281   
282    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
283    my @target_file_lines = &read_file_lines($target_file_path);
284    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
285   
286    # Filter out any automatically translated chunks
287    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
288        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
289            delete $source_file_key_to_line_mapping{$chunk_key};
290            delete $target_file_key_to_line_mapping{$chunk_key};
291        }
292    }
293   
294    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
295    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
296    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
297    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
298   
299    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
300    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
301 
302
303    # Chunks needing updating are those in the target file that have been more recently edited in the source file
304    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
305    my @uptodate_target_file_keys = ();
306    foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
307        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
308        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
309       
310        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
311       
312        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
313            # &log_message("Chunk with key $chunk_key needs updating.");
314            push(@uptodate_target_file_keys, $chunk_key);
315        }
316    }
317
318    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
319
320    return $xml_response;
321}
322
323
324sub get_first_n_chunks_requiring_work
325{
326    # The code of the target language (ensure it is lowercase)
327    my $target_language_code = lc(shift(@_));
328    # The key of the file to translate (ensure it is lowercase)
329    my $translation_file_key = lc(shift(@_));
330    # The number of chunks to return (defaults to one if not specified)
331    my $num_chunks_to_return = shift(@_) || "1";
332   
333    # Check that the necessary arguments were supplied
334    if (!$target_language_code || !$translation_file_key) {
335        &throw_fatal_error("Missing command argument.");
336    }
337   
338    # Get (and check) the translation configuration
339    my ($source_file, $target_file, $translation_file_type)
340    = &get_translation_configuration($target_language_code, $translation_file_key);
341
342    # Parse the source language and target language files
343    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
344    my @source_file_lines = &read_file_lines($source_file_path);
345    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
346   
347    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
348    my @target_file_lines = &read_file_lines($target_file_path);
349    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
350   
351    # Filter out any automatically translated chunks
352    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
353        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
354            delete $source_file_key_to_line_mapping{$chunk_key};
355            delete $target_file_key_to_line_mapping{$chunk_key};
356        }
357    }
358   
359    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
360    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
361    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
362    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
363   
364    # Determine the target file chunks requiring translation
365    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
366    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
367   
368    # Determine the target file chunks requiring updating
369    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
372    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
373   
374    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
375   
376    return $xml_response;
377}
378
379
380sub get_language_status
381{
382    # The code of the target language (ensure it is lowercase)
383    my $target_language_code = lc(shift(@_));
384   
385    # Check that the necessary arguments were supplied
386    if (!$target_language_code) {
387        &throw_fatal_error("Missing command argument.");
388    }
389   
390    # Form an XML response to the command
391    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
392    $xml_response .= "<GTIResponse>\n";
393    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
394   
395    foreach my $translation_file (@$gti_translation_files) {   
396        my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
397        my $target_file_name = "";
398       
399        if ($translation_file->{'key'} eq "gs3interface") {
400            my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
401            &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
402           
403            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
404            my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
405           
406            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
407            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
408            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
409            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
410        }
411        else {
412            # Get (and check) the translation configuration
413            my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
414            $target_file_name = $target_file;
415           
416            # Parse the source language and target language files
417            my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
418            my @source_file_lines = &read_file_lines($source_file_path);
419            my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
420           
421            my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
422            my @target_file_lines = &read_file_lines($target_file_path);
423            my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
424           
425            # Filter out any automatically translated chunks
426            foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
427                if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
428                    delete $source_file_key_to_line_mapping{$chunk_key};
429                    delete $target_file_key_to_line_mapping{$chunk_key};
430                }
431            }
432           
433            my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
434            my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
435           
436            # Determine the target file chunks requiring translation
437            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
438           
439            # Determine the target file chunks requiring updating
440            my @target_file_keys_requiring_updating = ();
441            if (-e $target_file_path) {
442                my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
443                my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
444                @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
445            }
446           
447            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
448            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
449            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
450            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
451        }
452       
453        &log_message("Status of " . $translation_file->{'key'});
454        &log_message("Number of source chunks: " . $num_source_chunks);
455        &log_message("Number of target chunks: " . $num_target_chunks);
456        &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
457        &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
458       
459        $xml_response .= "    <TranslationFile"
460        . " key=\"" . $translation_file->{'key'} . "\""
461        . " target_file_path=\"" . $target_file_name . "\""
462        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
463        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
464        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
465    }
466   
467    $xml_response .= "  </LanguageStatus>\n";
468   
469    $xml_response .= "</GTIResponse>\n";
470    return $xml_response;
471}
472
473
474sub search_chunks
475{
476    # The code of the target language (ensure it is lowercase)
477    my $target_language_code = lc(shift(@_));
478    # The key of the file to translate (ensure it is lowercase)
479    my $translation_file_key = lc(shift(@_));
480    # The query string
481    my $query_string = join(' ', @_);
482   
483    # Check that the necessary arguments were supplied
484    if (!$target_language_code || !$translation_file_key || !$query_string) {
485        &throw_fatal_error("Missing command argument.");
486    }
487   
488    my ($source_file, $target_file, $translation_file_type) = ();
489    my %source_file_key_to_text_mapping = ();
490    my %target_file_key_to_text_mapping = ();
491   
492   
493    if ($translation_file_key ne "gs3interface") {
494        # Get (and check) the translation configuration
495        ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
496       
497        # Parse the source language and target language files
498        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
499        my @source_file_lines = &read_file_lines($source_file_path);
500        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
501       
502        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
503        my @target_file_lines = &read_file_lines($target_file_path);
504        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
505       
506        # Filter out any automatically translated chunks
507        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
508            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
509                delete $source_file_key_to_line_mapping{$chunk_key};
510                delete $target_file_key_to_line_mapping{$chunk_key};
511            }
512        }
513       
514        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
515        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
516    }
517    else {
518        # Not needed in this case
519        my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
520        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
521        \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
522    }
523   
524    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
525    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
526   
527    # Determine the target file chunks matching the query
528    my @target_file_keys_matching_query = ();
529    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
530        my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
531        if ($target_file_text =~ /$query_string/i) {
532            # &log_message("Chunk with key $chunk_key matches query.");
533            push(@target_file_keys_matching_query, $chunk_key);
534        }
535    }
536   
537    # Form an XML response to the command
538    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
539    $xml_response .= "<GTIResponse>\n";
540   
541    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
542    foreach my $chunk_key (@target_file_keys_matching_query) {
543        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
544       
545        $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
546        $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
547        $xml_response .= "    </Chunk>\n";
548    }
549    $xml_response .= "  </ChunksMatchingQuery>\n";
550   
551    $xml_response .= "</GTIResponse>\n";
552    return $xml_response;
553}
554
555
556sub submit_translations
557{
558    # The code of the target language (ensure it is lowercase)
559    my $target_language_code = lc(shift(@_));
560    # The key of the file to translate (ensure it is lowercase)
561    my $translation_file_key = lc(shift(@_));
562    # The username of the translation submitter
563    my $submitter_username = shift(@_);
564    # Whether to submit a target chunk even if it hasn't changed
565    my $force_submission_flag = shift(@_);
566   
567    # Check that the necessary arguments were supplied
568    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
569        &log_message("Fatal error (but cannot be thrown): Missing command argument.");
570        die "\n";
571    }
572   
573    my %source_file_key_to_text_mapping = ();
574    my %source_file_key_to_gti_comment_mapping = ();
575    my %target_file_key_to_text_mapping = ();
576    my %target_file_key_to_gti_comment_mapping = ();
577   
578    my (@source_file_lines, @target_file_lines) = ();
579    my ($source_file, $target_file, $translation_file_type);
580   
581   
582    if ($translation_file_key ne "gs3interface") {
583        # Get (and check) the translation configuration
584        ($source_file, $target_file, $translation_file_type)
585        = &get_translation_configuration($target_language_code, $translation_file_key);
586       
587        # Parse the source language and target language files
588        @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
589        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
590        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
591        %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
592       
593        @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
594        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
595        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
596        %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
597    }
598    else {
599        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
600        \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
601    }
602    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
603    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
604   
605    # Submission date
606    my $day = (localtime)[3];
607    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
608    my $year = (localtime)[5] + 1900;
609    my $submission_date = "$day-$month-$year";
610   
611    open(SUBMISSION, "-");
612    my @submission_lines = <SUBMISSION>;
613    close(SUBMISSION);
614   
615    # Remove any nasty carriage returns
616    # &log_message("Submission:");
617    foreach my $submission_line (@submission_lines) {
618        $submission_line =~ s/\r$//;
619        #&log_message("  $submission_line");
620    }
621   
622    my %source_file_key_to_submission_mapping = ();
623    my %target_file_key_to_submission_mapping = ();
624    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
625        # Read source file part of submission
626        if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
627            my $chunk_key = $1;
628           
629            # Read the source file text
630            my $source_file_chunk_text = "";
631            $i++;
632            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
633                $source_file_chunk_text .= $submission_lines[$i];
634                $i++;
635            }
636            $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
637            $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
638           
639            #&log_message("Source file key: $chunk_key");
640            #&log_message("Source file text: $source_file_chunk_text");
641            $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
642        }
643       
644        # Read target file part of submission
645        if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
646            my $chunk_key = $1;
647           
648            # Read the target file text
649            my $target_file_chunk_text = "";
650            $i++;
651            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
652                $target_file_chunk_text .= $submission_lines[$i];
653                $i++;
654            }
655            $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
656            $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
657           
658            #&log_message("Target file key: $chunk_key");
659            #&log_message("Target file text: $target_file_chunk_text");
660            $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
661        }
662    }
663   
664    # -----------------------------------------
665    #   Validate the translation submissions
666    # -----------------------------------------
667   
668    # Check that the translations are valid
669    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
670        # Make sure the submitted chunk still exists in the source file
671        if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
672            &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
673            delete $source_file_key_to_submission_mapping{$chunk_key};
674            delete $target_file_key_to_submission_mapping{$chunk_key};
675            next;
676        }
677       
678        # Make sure the submitted source chunk matches the source file chunk
679        if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
680        #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
681                    #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
682                #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
683
684            &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
685            &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
686            &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
687            delete $source_file_key_to_submission_mapping{$chunk_key};
688            delete $target_file_key_to_submission_mapping{$chunk_key};
689            next;
690        }
691    }
692   
693    # Apply the submitted translations
694    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
695        # Only apply the submission if it is a change, unless -force_submission has been specified
696        if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
697            $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
698            $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
699        }
700    }
701   
702    if ($translation_file_key ne "gs3interface") {
703        eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
704    } else {
705        eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
706    }
707}
708
709
710sub create_glihelp_zip_file
711{
712    my $target_language_code = shift(@_);
713    my $translation_file_key = "glihelp";
714   
715    &log_message("Creating GLI Help zip file for $target_language_code");
716   
717    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
718   
719    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
720    my $oldclasspath = $classpath;
721    if ( ! -e $classpath) {
722    $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
723    }
724    if ( ! -e $classpath) {
725        &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
726    }
727
728   
729    my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
730    my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
731    my $os = $^O;
732    my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
733    my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
734    $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
735
736    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
737    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
738   
739    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
740    if ( ! -e $gen_many_html_xsl_filepath) {
741        &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
742    }
743   
744    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
745    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
746   
747    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
748    $target_file_directory = $target_file_directory."/";
749   
750    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
751
752    # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
753    my ($tailname, $glihelp_lang_dir, $suffix) =  &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");   
754    if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
755
756    # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
757    my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
758    my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
759    &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
760
761    # In gli/help/<newlang>/help.xml, replace all occurrences of
762    # <Text id="1">This text in en will be removed for new langcode</Text>
763    # with <!-- Missing translation: 1 -->
764    open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
765    my $help_xml_contents;
766    # Read in the entire contents of the file in one hit
767    sysread(FIN, $help_xml_contents, -s FIN);
768    close(FIN);
769   
770    $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
771
772    open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
773    print FOUT $help_xml_contents;
774    close(FOUT);
775    }
776
777    my $perl_exec = &util::get_perl_exec();
778    my $java_exec = "java";
779    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
780        $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
781    }
782
783    #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
784    my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
785    #&throw_fatal_error("RAN gti command: $cmd");
786    my $response = `$cmd`;
787
788    #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
789    $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
790    $response = `$cmd`;
791
792    # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
793    my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
794    if(!&FileUtils::directoryExists($tmpdir)) {
795    &FileUtils::makeDirectory($tmpdir);
796    }
797    #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";   
798    my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
799    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
800
801    $response = `$cmd`;
802}
803
804
805sub get_translation_configuration
806{
807    # Get the code of the target language
808    my $target_language_code = shift(@_);
809    # Get the key of the file to translate
810    my $translation_file_key = shift(@_);
811   
812    # Read the translation data from the gti.cfg file
813    my ($source_file, $target_file, $translation_file_type) =
814    &get_translation_data_for($target_language_code, $translation_file_key);
815   
816    # Check that the file to translate is defined in the gti.cfg file
817    if (!$source_file || !$target_file || !$translation_file_type) {
818        &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
819    }
820   
821    # Check that the source file exists
822    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
823    if (!-e $source_file_path) {
824        &throw_fatal_error("Source file $source_file_path does not exist.");
825    }
826   
827    # Check that the source file is up to date
828    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
829    # unless ($translation_file_is_not_in_cvs) {
830    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
831    my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
832    if ($source_file_cvs_status =~ /^C /) {
833        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
834    }
835    if ($source_file_cvs_status =~ /^M /) {
836        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
837    }
838    # }
839   
840    return ($source_file, $target_file, $translation_file_type);
841}
842
843
844sub get_translation_data_for
845{
846    my ($target_language_code, $translation_file_key) = @_;
847   
848    foreach my $translation_file (@$gti_translation_files) {
849        # If this isn't the correct translation file, move onto the next one
850        next if ($translation_file_key ne $translation_file->{'key'});
851       
852        # Resolve the target language file
853        my $target_language_file = $translation_file->{'target_file'};
854        if ($target_language_file =~ /(\{.+\;.+\})/) {
855            my $unresolved_target_language_file_part = $1;
856           
857            # Check for a special case for the target language code
858            if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
859                my $resolved_target_language_file_part = $2;
860                $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
861            }
862            # Otherwise use the last part as the default value
863            else {
864                my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
865            $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
866        }
867    }
868   
869    # Resolve instances of {iso_639_1_target_language_name}
870    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
871    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
872    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
873   
874    # Resolve instances of {target_language_code}
875    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
876   
877    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
878}
879
880return ();
881}
882
883
884sub read_file_lines
885{
886    my ($file_path) = @_;
887   
888    if (!open(FILE_IN, "<$file_path")) {
889        &log_message("Note: Could not open file $file_path.");
890        return ();
891    }
892    my @file_lines = <FILE_IN>;
893    close(FILE_IN);
894   
895    return @file_lines;
896}
897
898
899sub build_key_to_line_mapping
900{
901    my ($file_lines, $translation_file_type) = @_;
902    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
903}
904
905
906sub build_key_to_text_mapping
907{
908    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
909   
910    my %key_to_text_mapping = ();
911    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
912        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
913        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
914       
915        my $chunk_text = @$file_lines[$chunk_starting_line];
916        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
917            $chunk_text .= @$file_lines[$l];
918        }
919       
920        # Map from chunk key to text
921        eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
922    }
923   
924    return %key_to_text_mapping;
925}
926
927
928sub build_key_to_last_update_date_mapping
929{
930    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
931   
932    # If the files aren't in CVS then we can't tell anything about what needs updating
933    # return () if ($translation_file_is_not_in_cvs);
934   
935    # Build a mapping from key to CVS date
936    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
937    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
938   
939    # Build a mapping from key to comment date
940    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
941   
942    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
943    my %key_to_last_update_date_mapping = ();
944    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
945        # Use the CVS date as a starting point
946        my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
947        $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
948       
949        # If a comment date exists and it is after the CVS date, use that instead
950        # need to convert the comment date format to SVN format
951        my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
952        if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
953            my $chunk_comment_date = $1;           
954            if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
955                $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;         
956            }
957        }
958    }
959   
960    return %key_to_last_update_date_mapping;
961}
962
963
964sub build_key_to_cvs_date_mapping
965{
966    my ($filename, $translation_file_type) = @_;
967   
968    # Use SVN to annotate each line of the file with the date it was last edited
969    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
970    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
971   
972    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
973   
974    my @cvs_annotated_file_lines_date = ();
975    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
976        # Extract the date from the SVN annotation at the front
977        # svn format : 2007-07-16
978        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
979       
980        push(@cvs_annotated_file_lines_date, $1);
981       
982        # trim extra date information in svn annotation format
983        # 15:42:49 +1200 (Wed, 21 Jun 2006)
984        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
985    }   
986   
987    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
988    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
989   
990    my %key_to_cvs_date_mapping = ();
991    foreach my $chunk_key (keys(%key_to_line_mapping)) {
992        my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
993        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
994       
995        # Find the date this chunk was last edited, from the CVS annotation
996        my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
997        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
998            if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
999                # This part of the chunk has been updated more recently
1000                $chunk_date = $cvs_annotated_file_lines_date[$l];
1001               
1002            }
1003        }
1004       
1005        # Map from chunk key to CVS date
1006        $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1007    }
1008   
1009    return %key_to_cvs_date_mapping;
1010}
1011
1012
1013sub build_key_to_gti_comment_mapping
1014{
1015    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1016   
1017    my %key_to_gti_comment_mapping = ();
1018    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1019        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1020        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1021       
1022        my $chunk_text = @$file_lines[$chunk_starting_line];
1023        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1024            $chunk_text .= @$file_lines[$l];
1025        }
1026       
1027        # Map from chunk key to GTI comment
1028        my $chunk_gti_comment;
1029        eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1030        $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1031    }
1032   
1033    return %key_to_gti_comment_mapping;
1034}
1035
1036
1037sub determine_chunks_requiring_translation
1038{
1039    my $source_file_key_to_text_mapping = shift(@_);
1040    my $target_file_key_to_text_mapping = shift(@_);
1041   
1042    # Chunks needing translation are those in the source file with no translation in the target file
1043    my @target_file_keys_requiring_translation = ();
1044    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1045        if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1046            # &log_message("Chunk with key $chunk_key needs translating.");
1047            push(@target_file_keys_requiring_translation, $chunk_key);
1048        }
1049    }
1050   
1051    return @target_file_keys_requiring_translation;
1052}
1053
1054
1055sub determine_chunks_requiring_updating
1056{
1057    my $source_file_key_to_last_update_date_mapping = shift(@_);
1058    my $target_file_key_to_last_update_date_mapping = shift(@_);
1059   
1060    # Chunks needing updating are those in the target file that have been more recently edited in the source file
1061    my @target_file_keys_requiring_updating = ();
1062    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1063        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1064        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1065       
1066        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1067
1068        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1069            # &log_message("Chunk with key $chunk_key needs updating.");
1070                # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1071            push(@target_file_keys_requiring_updating, $chunk_key);
1072        }
1073    }
1074   
1075    return @target_file_keys_requiring_updating;
1076}
1077
1078
1079sub is_chunk_automatically_translated
1080{
1081    my ($chunk_key, $translation_file_type) = @_;
1082    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1083}
1084
1085
1086sub make_text_xml_safe
1087{
1088    my $text = shift(@_);
1089    $text =~ s/\&/\&amp\;/g;
1090    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1091    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1092    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1093    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1094    $text =~ s/</\&lt\;/g;
1095    $text =~ s/>/\&gt\;/g;
1096    return $text;
1097}
1098
1099
1100sub unmake_text_xml_safe
1101{
1102    my $text = shift(@_);
1103    $text =~ s/\&lt\;/</g;
1104    $text =~ s/\&gt\;/>/g;
1105    $text =~ s/\&amp\;/\&/g;
1106    return $text;
1107}
1108
1109
1110# Returns 1 if $date1 is after $date2, 0 otherwise
1111sub is_date_after_cvs
1112{
1113    my ($date1, $date2) = @_;
1114    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
1115    "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1116   
1117    if(!defined $date1) {
1118        return 1;
1119    }
1120   
1121    my @date1parts = split(/-/, $date1);
1122    my @date2parts = split(/-/, $date2);
1123   
1124    # Compare year - nasty because we have rolled over into a new century
1125    my $year1 = $date1parts[2];
1126    if ($year1 < 80) {
1127        $year1 += 2000;
1128    }
1129    my $year2 = $date2parts[2];
1130    if ($year2 < 80) {
1131        $year2 += 2000;
1132    }
1133   
1134    # Compare year
1135    if ($year1 > $year2) {
1136        return 1;
1137    }
1138    elsif ($year1 == $year2) {
1139        # Year is the same, so compare month
1140        if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1141            return 1;
1142        }
1143        elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1144            # Month is the same, so compare day
1145            if ($date1parts[0] > $date2parts[0]) {
1146                return 1;
1147            }
1148        }
1149    }
1150   
1151    return 0;
1152}
1153
1154sub is_date_after
1155{
1156    my ($date1, $date2) = @_;
1157   
1158    if(!defined $date1) {
1159        return 1;
1160    }
1161    if(!defined $date2) {
1162        return 0;
1163    }
1164   
1165    # 16-Aug-2006
1166    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1167        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1168        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1169        $date1=$3 . "-" . $months{$2} . "-" . $1;
1170        # print "** converted date1: $date1\n";
1171    }
1172    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1173        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1174        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1175        $date2=$3 . "-" . $months{$2} . "-" . $1;
1176        # print "** converted date2: $date2\n";
1177    }
1178   
1179   
1180    # 2006-08-16
1181    my @date1parts = split(/-/, $date1);
1182    my @date2parts = split(/-/, $date2);
1183   
1184    # Compare year
1185    if ($date1parts[0] > $date2parts[0]) {
1186        return 1;
1187    }
1188    elsif ($date1parts[0] == $date2parts[0]) {
1189        # Year is the same, so compare month
1190        if ($date1parts[1] > $date2parts[1]) {
1191            return 1;
1192        }
1193        elsif ($date1parts[1] == $date2parts[1]) {
1194            # Month is the same, so compare day
1195            if ($date1parts[2] > $date2parts[2]) {
1196                return 1;
1197            }
1198        }
1199    }   
1200   
1201    return 0;
1202}
1203
1204
1205sub create_xml_response_for_chunks_requiring_work
1206{
1207    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1208   
1209    # Form an XML response to the command
1210    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1211    $xml_response .= "<GTIResponse>\n";
1212    $xml_response .= "  <TranslationFile"
1213    . " key=\"" . $translation_file_key . "\""
1214    . " target_file_path=\"" . $target_file . "\""
1215    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1216    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1217    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1218   
1219    # Do chunks requiring translation first
1220    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1221        $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1222    }
1223    else {
1224        $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1225    }
1226   
1227    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1228    foreach my $chunk_key (@sorted_chunk_keys) {
1229        last if ($num_chunks_to_return == 0);
1230       
1231        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1232        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1233       
1234        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1235        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1236        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1237        $xml_response .= "    </Chunk>\n";
1238       
1239        $num_chunks_to_return--;
1240    }
1241   
1242    $xml_response .= "  </ChunksRequiringTranslation>\n";
1243   
1244    # Then do chunks requiring updating
1245    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1246        $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1247    }
1248    else {
1249        $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1250    }
1251   
1252    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1253    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1254    foreach my $chunk_key (@sorted_chunk_keys) {
1255        last if ($num_chunks_to_return == 0);
1256       
1257        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1258        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1259        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1260        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1261       
1262        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1263        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1264        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1265        $xml_response .= "    </Chunk>\n";
1266       
1267        $num_chunks_to_return--;
1268    }
1269   
1270    $xml_response .= "  </ChunksRequiringUpdating>\n";
1271   
1272    $xml_response .= "</GTIResponse>\n";
1273   
1274    return $xml_response;
1275}
1276
1277sub create_xml_response_for_uptodate_chunks
1278{
1279    my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1280   
1281    # Form an XML response to the command
1282    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1283    $xml_response .= "<GTIResponse>\n";
1284    $xml_response .= "  <TranslationFile"
1285    . " key=\"" . $translation_file_key . "\""
1286    . " target_file_path=\"" . $target_file . "\""
1287    . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1288   
1289   
1290    # Then do chunks requiring updating
1291    $xml_response .= "  <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1292   
1293   
1294    # foreach my $chunk_key (@uptodate_target_file_keys) {
1295    my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1296    foreach my $chunk_key (@sorted_chunk_keys) {
1297       
1298        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1299        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1300        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1301        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1302       
1303        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1304        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1305        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1306        $xml_response .= "    </Chunk>\n";
1307
1308    }
1309   
1310    $xml_response .= "  </UptodateChunks>\n";
1311   
1312    $xml_response .= "</GTIResponse>\n";
1313   
1314    return $xml_response;
1315}
1316
1317sub create_xml_response_for_all_chunks
1318{
1319    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1320   
1321    # Form an XML response to the command
1322    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1323    $xml_response .= "<GTIResponse>\n";
1324    $xml_response .= "  <TranslationFile"
1325    . " key=\"" . $translation_file_key . "\""
1326    . " target_file_path=\"" . $target_file . "\"\/>\n";
1327   
1328    # Do all the chunks
1329    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1330   
1331    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1332    foreach my $chunk_key (@sorted_chunk_keys) {
1333        my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1334        my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1335       
1336        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1337        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1338        if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1339            my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1340            my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1341            $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1342        }
1343        else {
1344            $xml_response .= "      <TargetFileText></TargetFileText>\n";
1345        }
1346       
1347        $xml_response .= "    </Chunk>\n";
1348    }
1349    $xml_response .= "  </Chunks>\n";
1350   
1351    $xml_response .= "</GTIResponse>\n";
1352    return $xml_response;
1353}
1354
1355
1356
1357# ==========================================================================================
1358#   MACROFILE FUNCTIONS
1359
1360sub build_key_to_line_mapping_for_macrofile
1361{
1362    my (@file_lines) = @_;
1363   
1364    my $macro_package;
1365    my %chunk_key_to_line_mapping = ();
1366    # Process the contents of the file, line by line
1367    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1368        my $line = $file_lines[$i];
1369        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1370       
1371        # Check if a new package is being defined
1372        if ($line =~ m/^package\s+(.+)/) {
1373            $macro_package = $1;
1374        }
1375       
1376        # Line contains a macro name
1377        elsif ($line =~ m/^(_\w+_)/) {
1378            my $macro_key = $1;
1379            $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1380           
1381            # While there is still text of the macro to go...
1382            my $startline = $i;
1383            while ($line !~ /\}$/) {
1384                $i++;
1385                if ($i == scalar(@file_lines)) {
1386                    &throw_fatal_error("Could not find end of macro $macro_key.");
1387                }
1388                $line = $file_lines[$i];
1389                $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1390            }
1391       
1392        # The chunk key consists of the package name and the macro key
1393        my $chunk_key = $macro_package . "." . $macro_key;
1394        # Map from chunk key to line
1395        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1396    }
1397   
1398    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1399    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1400    # The chunk key consists of package name and macro key
1401    my $chunk_key = $macro_package . "." . $1;
1402    # Map from chunk key to line
1403    $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1404}
1405}
1406
1407return %chunk_key_to_line_mapping;
1408}
1409
1410
1411sub import_chunk_from_macrofile
1412{
1413    my ($chunk_text) = @_;
1414   
1415    # Is this an icon macro??
1416    if ($chunk_text =~ /^\#\# (.*)/) {
1417        # Extract image macro text
1418        $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1419        $chunk_text = $1;
1420   
1421    # Remove enclosing quotes
1422    $chunk_text =~ s/^\"//;
1423    $chunk_text =~ s/\"$//;
1424}
1425
1426# No, so it must be a text macro
1427else {
1428    # Remove macro key
1429    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1430   
1431    # Remove language specifier
1432    $chunk_text =~ s/^\[l=.*\](\s*)//;
1433   
1434    # Remove braces enclosing text
1435    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1436}
1437
1438return $chunk_text;
1439}
1440
1441
1442sub get_macrofile_chunk_gti_comment
1443{
1444    my ($chunk_text) = @_;
1445   
1446    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1447    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1448        return $1;
1449}
1450
1451return undef;
1452}
1453
1454
1455sub is_macrofile_chunk_automatically_translated
1456{
1457    my ($chunk_key) = @_;
1458   
1459    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1460    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1461        return 1;
1462    }
1463   
1464    return 0;
1465}
1466
1467
1468# Use the source file to generate a target file that is formatted the same
1469sub write_translated_macrofile
1470{
1471    my $source_file = shift(@_);  # Not used
1472    my @source_file_lines = @{shift(@_)};
1473    my $source_file_key_to_text_mapping = shift(@_);
1474    my $target_file = shift(@_);
1475    my @target_file_lines = @{shift(@_)};
1476    my $target_file_key_to_text_mapping = shift(@_);
1477    my $target_file_key_to_gti_comment_mapping = shift(@_);
1478    my $target_language_code = shift(@_);
1479   
1480    # Build a mapping from source file line to chunk key
1481    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1482    my %source_file_line_to_key_mapping = ();
1483    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1484        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1485    }
1486    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1487    my $source_file_line_number = 0;
1488   
1489    # Build a mapping from target file line to chunk key
1490    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1491    my %target_file_line_to_key_mapping = ();
1492    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1493        $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1494    }
1495    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1496   
1497    # Write the new target file
1498    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1499    if (!open(TARGET_FILE, ">$target_file_path")) {
1500        &throw_fatal_error("Could not write target file $target_file_path.");
1501    }
1502   
1503    # Use the header from the target file, to keep language and author information
1504    if (scalar(@target_file_line_keys) > 0) {
1505        my $target_file_line_number = 0;
1506        my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1507        while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1508            my $target_file_line = $target_file_lines[$target_file_line_number];
1509            last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1510                print TARGET_FILE $target_file_line;
1511            $target_file_line_number++;
1512        }
1513       
1514        $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1515    }
1516   
1517    # Model the new target file on the source file, with the target file translations
1518    foreach my $line_key (@source_file_line_keys) {
1519        # Fill in the gaps before this chunk starts
1520        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1521        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1522        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1523            print TARGET_FILE $source_file_lines[$source_file_line_number];
1524            $source_file_line_number++;
1525        }
1526        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1527       
1528        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1529        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1530        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1531       
1532        my $macrofile_key = $chunk_key;
1533        $macrofile_key =~ s/^(.+?)\.//;
1534       
1535        # If no translation exists for this chunk, show this, and move on
1536        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1537            print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1538            next;
1539        }
1540       
1541        # Grab the source chunk text
1542        my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1543        for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1544            $source_file_chunk .= $source_file_lines[$l];
1545        }
1546       
1547        # Is this an icon macro??
1548        if ($source_file_chunk =~ /^\#\# (.*)/) {
1549            # Escape any newline and question mark characters so the source text is replaced correctly
1550            $source_file_chunk_text =~ s/\\/\\\\/g;
1551        $source_file_chunk_text =~ s/\?/\\\?/g;
1552       
1553        # Build the new target chunk from the source chunk
1554        my $target_file_chunk = $source_file_chunk;
1555        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1556        $target_file_chunk =~ s/(\s)*$//;
1557        print TARGET_FILE "$target_file_chunk";
1558    }
1559   
1560    # No, it is just a normal text macro
1561    else {
1562        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1563    }
1564   
1565    # Add the "updated" comment, if one exists
1566    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1567        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1568    }
1569    print TARGET_FILE "\n";
1570}
1571
1572close(TARGET_FILE);
1573}
1574
1575
1576sub sort_by_line
1577{
1578    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1579}
1580
1581
1582# ==========================================================================================
1583#   RESOURCE BUNDLE FUNCTIONS
1584
1585# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1586sub build_key_to_line_mapping_for_resource_bundle
1587{
1588    my (@file_lines) = @_;
1589   
1590    my %chunk_key_to_line_mapping = ();
1591
1592    my $chunk_key;
1593    my $startindex = -1;
1594
1595    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1596        my $line = $file_lines[$i];
1597        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1598       
1599        # Line contains a dictionary string
1600        if ($line =~ /^(\S+?)[:|=](.*)$/) {
1601            $chunk_key = $1;
1602            $startindex = $i;
1603        }       
1604        if ($startindex != -1) {
1605            if($line !~ m/\\$/) { # line finished
1606            # $i keeps track of the line at which this property (chunk_key) finishes
1607
1608            # Map from chunk key to line
1609            $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1610            $startindex = -1;
1611            $chunk_key = "";
1612            }
1613        }               
1614    }
1615   
1616    return %chunk_key_to_line_mapping;
1617}
1618
1619
1620sub import_chunk_from_resource_bundle
1621{
1622    my ($chunk_text) = @_;
1623   
1624    # Simple: just remove string key
1625    $chunk_text =~ s/^(\S+?)[:|=](\s*)//s;
1626    $chunk_text =~ s/(\s*)$//s;  # Remove any nasty whitespace, carriage returns etc.
1627    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1628   
1629    return $chunk_text;
1630}
1631
1632
1633sub get_resource_bundle_chunk_gti_comment
1634{
1635    my ($chunk_text) = @_;
1636   
1637    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1638    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1639        return $1;
1640}
1641
1642return undef;
1643}
1644
1645
1646sub is_resource_bundle_chunk_automatically_translated
1647{
1648    # No resource bundle chunks are automatically translated
1649    return 0;
1650}
1651
1652
1653sub write_translated_resource_bundle
1654{
1655    my $source_file = shift(@_);  # Not used
1656    my @source_file_lines = @{shift(@_)};
1657    my $source_file_key_to_text_mapping = shift(@_);
1658    my $target_file = shift(@_);
1659    my @target_file_lines = @{shift(@_)};  # Not used
1660    my $target_file_key_to_text_mapping = shift(@_);
1661    my $target_file_key_to_gti_comment_mapping = shift(@_);
1662    my $target_language_code = shift(@_);  # Not used
1663   
1664    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1665    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1666    my %source_file_line_to_key_mapping = ();
1667    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1668        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1669    }
1670   
1671    # Write the new target file
1672    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1673    if (!open(TARGET_FILE, ">$target_file_path")) {
1674        &throw_fatal_error("Could not write target file $target_file_path.");
1675    }
1676   
1677    # Model the new target file on the source file, with the target file translations
1678    my $source_file_line_number = 0;
1679    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1680        # Fill in the gaps before this chunk starts
1681        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1682        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1683        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1684            print TARGET_FILE $source_file_lines[$source_file_line_number];
1685            $source_file_line_number++;
1686        }
1687        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1688       
1689        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1690        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1691        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1692       
1693        # If no translation exists for this chunk, show this, and move on
1694        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1695            print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1696            next;
1697        }
1698       
1699        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1700        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1701            print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1702        }
1703        print TARGET_FILE "\n";
1704    }
1705   
1706    close(TARGET_FILE);
1707}
1708
1709
1710# ==========================================================================================
1711#   GREENSTONE XML FUNCTIONS
1712
1713sub build_key_to_line_mapping_for_greenstone_xml
1714{
1715    my (@file_lines) = @_;
1716   
1717    my %chunk_key_to_line_mapping = ();
1718    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1719        my $line = $file_lines[$i];
1720        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1721       
1722        # Line contains a string to translate
1723        if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1724            my $chunk_key = $1;
1725            $line =~ s/\s*$//;  # Remove any nasty whitespace
1726            $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1727           
1728            # While there is still text of the string to go...
1729            my $startline = $i;
1730            while ($line !~ /<\/Text>$/) {
1731                $i++;
1732                if ($i == scalar(@file_lines)) {
1733                    &throw_fatal_error("Could not find end of string $chunk_key.");
1734                }
1735                $line = $file_lines[$i];
1736                $line =~ s/\s*$//;  # Remove any nasty whitespace
1737                $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1738            }
1739           
1740            # Map from chunk key to line
1741            if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1742                $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1743            }
1744            else {
1745                &throw_fatal_error("Duplicate key $chunk_key.");
1746            }
1747        }
1748    }
1749   
1750    return %chunk_key_to_line_mapping;
1751}
1752
1753
1754sub import_chunk_from_greenstone_xml
1755{
1756    my ($chunk_text) = @_;
1757   
1758    # Simple: just remove the Text tags
1759    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1760    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1761    $chunk_text =~ s/<\/Text>$//;
1762   
1763    return $chunk_text;
1764}
1765
1766
1767sub get_greenstone_xml_chunk_gti_comment
1768{
1769    my ($chunk_text) = @_;
1770   
1771    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1772    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1773        return $1;
1774    }
1775   
1776    return undef;
1777}
1778
1779
1780sub is_greenstone_xml_chunk_automatically_translated
1781{
1782    # No greenstone XML chunks are automatically translated
1783    return 0;
1784}
1785
1786
1787sub write_translated_greenstone_xml
1788{
1789    my $source_file = shift(@_);  # Not used
1790    my @source_file_lines = @{shift(@_)};
1791    my $source_file_key_to_text_mapping = shift(@_);
1792    my $target_file = shift(@_);
1793    my @target_file_lines = @{shift(@_)};  # Not used
1794    my $target_file_key_to_text_mapping = shift(@_);
1795    my $target_file_key_to_gti_comment_mapping = shift(@_);
1796    my $target_language_code = shift(@_);  # Not used
1797   
1798    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1799    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1800    my %source_file_line_to_key_mapping = ();
1801    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1802        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1803    }
1804   
1805    # Write the new target file
1806    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1807    if (!open(TARGET_FILE, ">$target_file_path")) {
1808        &throw_fatal_error("Could not write target file $target_file_path.");
1809    }
1810   
1811    # Model the new target file on the source file, with the target file translations
1812    my $source_file_line_number = 0;
1813    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1814        # Fill in the gaps before this chunk starts
1815        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1816        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1817        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1818            print TARGET_FILE $source_file_lines[$source_file_line_number];
1819            $source_file_line_number++;
1820        }
1821        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1822       
1823        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1824        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1825        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1826        $target_file_chunk_text =~ s/(\n)*$//g;
1827       
1828        # If no translation exists for this chunk, show this, and move on
1829        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1830            print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1831            next;
1832        }
1833       
1834        print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1835        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1836            my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1837            $chunk_gti_comment =~ s/^Updated //;
1838            print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1839        }
1840        print TARGET_FILE "\n";
1841    }
1842   
1843    # Fill in the end of the file
1844    while ($source_file_line_number < scalar(@source_file_lines)) {
1845        print TARGET_FILE $source_file_lines[$source_file_line_number];
1846        $source_file_line_number++;
1847    }
1848   
1849    close(TARGET_FILE);
1850}
1851
1852
1853# ==========================================================================================
1854#   GREENSTONE3 FUNCTIONS
1855
1856sub get_all_chunks_for_gs3
1857{
1858    # The code of the target language (ensure it is lowercase)
1859    my $target_language_code = lc(shift(@_));
1860    my $translation_file_key = lc(shift(@_));
1861   
1862    # Check that the necessary arguments were supplied
1863    if (!$target_language_code) {
1864        &throw_fatal_error("Missing command argument.");
1865    }
1866   
1867    # Get (and check) the translation configuration
1868    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1869   
1870    my %source_files_key_to_text_mapping = ();
1871    my %target_files_key_to_text_mapping = ();
1872    my %source_files_key_to_last_update_date_mapping = ();
1873    my %target_files_key_to_last_update_date_mapping = ();
1874   
1875    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1876   
1877    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1878    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1879   
1880    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1881    return $xml_response;
1882}
1883
1884
1885sub get_first_n_chunks_requiring_work_for_gs3
1886{
1887    # The code of the target language (ensure it is lowercase)
1888    my $target_language_code = lc(shift(@_));
1889    # The key of the file to translate (ensure it is lowercase)
1890    my $translation_file_key = lc(shift(@_));
1891    # The number of chunks to return (defaults to one if not specified)
1892    my $num_chunks_to_return = shift(@_) || "1";
1893   
1894    # Check that the necessary arguments were supplied
1895    if (!$target_language_code || !$translation_file_key) {
1896        &throw_fatal_error("Missing command argument.");
1897    }
1898
1899    my %source_files_key_to_text_mapping = ();
1900    my %target_files_key_to_text_mapping = ();
1901    my %source_files_key_to_last_update_date_mapping = ();
1902    my %target_files_key_to_last_update_date_mapping = ();
1903   
1904    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1905    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1906   
1907    # Determine the target file chunks requiring translation
1908    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1909    # Determine the target file chunks requiring updating
1910    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1911    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1912    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1913
1914    my $download_target_filepath = "";
1915
1916
1917    # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1918
1919    # if there is no copy of the language files for download, there's also no link to the spreadsheet
1920    # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1921    # interface *.properties files,and link to that zip as the file for offline translation.
1922    # Selecting only properties files for English and the language they're working on (if the last exists)
1923
1924    # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1925    # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1926
1927    my $sourcedir = "greenstone3";
1928    my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1929    my $tar_cmd = "tar -cvzf $zip";
1930
1931
1932    # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1933    my $curdir = `pwd`;
1934    chdir $gsdl_root_directory;
1935
1936    my $filelisting = "";
1937    foreach my $interface_file (@gs3_interface_files) {
1938
1939    my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
1940    my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
1941   
1942    $filelisting = "$filelisting $source_filepath";
1943    if(&FileUtils::fileExists($target_filepath)) {
1944        $filelisting = "$filelisting $target_filepath";
1945    }
1946    }
1947    $tar_cmd .= " $filelisting";
1948
1949    # tar command will overwrite the previous version, but want to check we've created it
1950    if(&FileUtils::fileExists($zip)) {
1951    &FileUtils::removeFiles($zip);
1952    }
1953
1954    #my $tar_result = system($tar_cmd); # works but then interface breaks
1955    `$tar_cmd`;
1956    my $tar_result = $?;
1957
1958    if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1959    $download_target_filepath = $zip;
1960    } else {
1961    &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1962    }
1963
1964    # change back to original working directory (cgi-bin/linux probably)
1965    chdir $curdir;
1966
1967    # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
1968
1969
1970    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
1971    \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1972    $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1973    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1974   
1975    return $xml_response;
1976}
1977
1978sub get_uptodate_chunks_for_gs3
1979{
1980    # The code of the target language (ensure it is lowercase)
1981    my $target_language_code = lc(shift(@_));
1982    # The key of the file to translate (ensure it is lowercase)
1983    my $translation_file_key = lc(shift(@_));
1984    # The number of chunks to return (defaults to one if not specified)
1985    my $num_chunks_to_return = shift(@_) || "1";
1986   
1987    # Check that the necessary arguments were supplied
1988    if (!$target_language_code || !$translation_file_key) {
1989        &throw_fatal_error("Missing command argument.");
1990    }
1991   
1992    my %source_files_key_to_text_mapping = ();
1993    my %target_files_key_to_text_mapping = ();
1994    my %source_files_key_to_last_update_date_mapping = ();
1995    my %target_files_key_to_last_update_date_mapping = ();
1996   
1997    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1998    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1999   
2000
2001    # Chunks needing updating are those in the target file that have been more recently edited in the source file
2002    # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2003    my @uptodate_target_file_keys = ();
2004    foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2005        my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2006        my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2007       
2008        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
2009       
2010        if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2011            # &log_message("Chunk with key $chunk_key needs updating.");
2012            push(@uptodate_target_file_keys, $chunk_key);
2013        }
2014    }
2015
2016    my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2017   
2018    return $xml_response;
2019}
2020
2021
2022sub build_gs3_configuration
2023{
2024    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2025    $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2026   
2027    my $source_file_directory = "greenstone3";  # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2028    my $translation_file_type = "resource_bundle";
2029   
2030    foreach my $interface_file_key (@gs3_interface_files) {
2031       
2032        &log_message("Greenstone 3 interface file: " . $interface_file_key);
2033       
2034        # Parse the source language and target language files
2035        my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2036        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2037        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2038        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2039        #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
2040       
2041        my %source_file_key_to_gti_comment_or_last_updated_mapping;
2042        if($get_gti_comments_not_last_updated) {
2043            %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
2044        } else {
2045            %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2046        }
2047
2048        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2049        my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2050        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2051        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2052        #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2053       
2054        my %target_file_key_to_gti_comment_or_last_updated_mapping;
2055        if($get_gti_comments_not_last_updated) {
2056            %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2057        } else {
2058            %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2059        }
2060       
2061       
2062        # Filter out any automatically translated chunks
2063        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2064            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2065                delete $source_file_key_to_line_mapping{$chunk_key};
2066                delete $target_file_key_to_line_mapping{$chunk_key};
2067            }
2068        }
2069       
2070        &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2071        &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2072       
2073        foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2074            my $global_chunk_key = "$interface_file_key.$chunk_key";
2075            $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2076            $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2077           
2078            if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2079                $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2080                $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2081            }
2082        }
2083    }
2084}
2085
2086
2087sub write_translated_gs3interface
2088{
2089    my $source_file_key_to_text_mapping = shift(@_);
2090    my $target_file_key_to_text_mapping = shift(@_);
2091    my $target_file_key_to_gti_comment_mapping = shift(@_);
2092    my $target_language_code = shift(@_);
2093   
2094    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2095   
2096    my %translated_interface_file_keys = ();
2097    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2098        $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2099        if (!defined $translated_interface_file_keys{$1}) {
2100            &log_message("Updated interface file: " . $1); 
2101            $translated_interface_file_keys{$1}="";
2102        }
2103    }
2104    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2105   
2106    my $source_file_directory = "greenstone3";   
2107   
2108    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2109       
2110        # Build a mapping from chunk key to source file line, and from source file line to chunk key
2111        my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2112        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2113        my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2114        my %source_file_line_to_key_mapping = ();
2115        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2116            $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2117        }
2118       
2119        # Write the new target file
2120        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2121        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2122        if (!open(TARGET_FILE, ">$target_file_path")) {
2123            &throw_fatal_error("Could not write target file $target_file_path.");
2124        }
2125       
2126        # Model the new target file on the source file, with the target file translations
2127        my $source_file_line_number = 0;
2128        foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2129            # Fill in the gaps before this chunk starts
2130            my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2131            my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2132            while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2133                print TARGET_FILE $source_file_lines[$source_file_line_number];
2134                $source_file_line_number++;
2135            }
2136            $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2137           
2138            my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2139            my $global_chunk_key = "$interface_file_key.$chunk_key";
2140            my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2141            my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2142           
2143            # If no translation exists for this chunk, show this, and move on
2144            if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2145                print TARGET_FILE "# -- Missing translation: $chunk_key\n";
2146                next;
2147            }
2148           
2149            print TARGET_FILE "$chunk_key:$target_file_chunk_text";
2150            if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2151                print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2152            }
2153            print TARGET_FILE "\n";
2154        }
2155       
2156        close(TARGET_FILE);
2157    }           
2158}
2159
2160&main(@ARGV);
Note: See TracBrowser for help on using the browser.