root/main/trunk/greenstone2/bin/script/gti.pl @ 24627

Revision 24627, 75.4 KB (checked in by anna, 9 years ago)

tidied up gti.pl

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39
40my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
41my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
42my $source_language_code = "en";  # This is non-negiotable
43
44my $gti_translation_files =
45[ # Greenstone macrofiles
46{ 'key' => "coredm",
47    'file_type' => "macrofile",
48    'source_file' => "macros/english.dm",
49    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
50
51{ 'key' => "auxdm",
52    'file_type' => "macrofile",
53    'source_file' => "macros/english2.dm",
54    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
55
56# GLI dictionary
57{ 'key' => "glidict",
58    'file_type' => "resource_bundle",
59    'source_file' => "gli/classes/dictionary.properties",
60    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
61
62# GLI help
63{ 'key' => "glihelp",
64    'file_type' => "greenstone_xml",
65    'source_file' => "gli/help/en/help.xml",
66    'target_file' => "gli/help/{target_language_code}/help.xml" },
67
68# Greenstone Perl modules
69{ 'key' => "perlmodules",
70    'file_type' => "resource_bundle",
71    'source_file' => "perllib/strings.properties",
72    'target_file' => "perllib/strings_{target_language_code}.properties" },
73
74# Greenstone tutorial exercises
75# { 'key' => "tutorials",
76# 'file_type' => "greenstone_xml",
77# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
78# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
79
80# new Greenstone.org
81{ 'key' => "greenorg",
82    'file_type' => "resource_bundle",
83    'source_file' => "greenstoneorg/website/classes/Gsc.properties",
84    'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
85}
86
87# { 'key' => "gs3interface",
88#'file_type' => "resource_bundle",
89#'source_file' => "greenstone3",
90#'target_file' => "greenstone3"
91#}
92];
93
94my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
95
96
97sub main
98{
99    # Get the command to process, and any arguments
100    my $gti_command = shift(@_);
101    my @gti_command_arguments = @_;
102    my $module = $_[1];
103   
104    # Open the GTI log file for appending, or write to STDERR if that fails
105    if (!open(GTI_LOG, ">>$gti_log_file")) {
106        open(GTI_LOG, ">&STDERR");
107    }
108   
109    # Log the command that launched this script
110    &log_message("Command: $0 @ARGV");
111   
112    # Check that a command was supplied
113    if (!$gti_command) {
114        &throw_fatal_error("Missing command.");
115    }     
116   
117    # Process the command
118    if ($gti_command =~ /^get-all-chunks$/i) {
119        # Check that GS3 interface is the target
120        if ($module eq "gs3interface") {       
121            print &get_all_chunks_for_gs3(@gti_command_arguments);
122        } else {
123            print &get_all_chunks(@gti_command_arguments);
124        }
125    }
126    elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
127        if ($module eq "gs3interface") {       
128            print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
129        } else {
130            print &get_first_n_chunks_requiring_work(@gti_command_arguments);
131        }
132    }
133    elsif ($gti_command =~ /^get-language-status$/i) {
134        print &get_language_status(@gti_command_arguments);       
135    }
136    elsif ($gti_command =~ /^search-chunks$/i) {
137        print &search_chunks(@gti_command_arguments);
138    }
139    elsif ($gti_command =~ /^submit-translations$/i) {
140        # This command cannot produce any output since it reads input
141        &submit_translations(@gti_command_arguments);
142    }
143    elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
144        # This command cannot produce any output since it reads input
145        &create_glihelp_zip_file(@gti_command_arguments);
146    }
147    else {
148        # The command was not recognized
149        &throw_fatal_error("Unknown command \"$gti_command\".");
150    }
151}
152
153
154sub throw_fatal_error
155{
156    my $error_message = shift(@_);
157   
158    # Write an XML error response
159    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
160    print "<GTIResponse>\n";
161    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
162    print "</GTIResponse>\n";
163   
164    # Log the error message, then die
165    &log_message("Error: $error_message");
166    die "\n";
167}
168
169
170sub log_message
171{
172    my $log_message = shift(@_);
173    print GTI_LOG time() . " -- " . $log_message . "\n";
174}
175
176
177sub get_all_chunks
178{
179    # The code of the target language (ensure it is lowercase)
180    my $target_language_code = lc(shift(@_));
181    # The key of the file to translate (ensure it is lowercase)
182    my $translation_file_key = lc(shift(@_));
183   
184    # Check that the necessary arguments were supplied
185    if (!$target_language_code || !$translation_file_key) {
186        &throw_fatal_error("Missing command argument.");
187    }
188   
189    # Get (and check) the translation configuration
190    my ($source_file, $target_file, $translation_file_type)
191    = &get_translation_configuration($target_language_code, $translation_file_key);
192   
193    # Parse the source language and target language files
194    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
195    my @source_file_lines = &read_file_lines($source_file_path);
196    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
197   
198    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
199    my @target_file_lines = &read_file_lines($target_file_path);
200    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
201   
202    # Filter out any automatically translated chunks
203    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
204        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
205            delete $source_file_key_to_line_mapping{$chunk_key};
206            delete $target_file_key_to_line_mapping{$chunk_key};
207        }
208    }
209   
210    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
211    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
212    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
213    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
214   
215    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
216    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
217   
218    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
219   
220    return $xml_response;
221}
222
223
224sub get_first_n_chunks_requiring_work
225{
226    # The code of the target language (ensure it is lowercase)
227    my $target_language_code = lc(shift(@_));
228    # The key of the file to translate (ensure it is lowercase)
229    my $translation_file_key = lc(shift(@_));
230    # The number of chunks to return (defaults to one if not specified)
231    my $num_chunks_to_return = shift(@_) || "1";
232   
233    # Check that the necessary arguments were supplied
234    if (!$target_language_code || !$translation_file_key) {
235        &throw_fatal_error("Missing command argument.");
236    }
237   
238    # Get (and check) the translation configuration
239    my ($source_file, $target_file, $translation_file_type)
240    = &get_translation_configuration($target_language_code, $translation_file_key);
241   
242    # Parse the source language and target language files
243    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
244    my @source_file_lines = &read_file_lines($source_file_path);
245    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
246   
247    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
248    my @target_file_lines = &read_file_lines($target_file_path);
249    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
250   
251    # Filter out any automatically translated chunks
252    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
253        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
254            delete $source_file_key_to_line_mapping{$chunk_key};
255            delete $target_file_key_to_line_mapping{$chunk_key};
256        }
257    }
258   
259    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
260    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
261    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
262    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
263   
264    # Determine the target file chunks requiring translation
265    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
266    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
267   
268    # Determine the target file chunks requiring updating
269    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
270    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
271    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
272    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
273   
274    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
275   
276    return $xml_response;
277}
278
279
280sub get_language_status
281{
282    # The code of the target language (ensure it is lowercase)
283    my $target_language_code = lc(shift(@_));
284   
285    # Check that the necessary arguments were supplied
286    if (!$target_language_code) {
287        &throw_fatal_error("Missing command argument.");
288    }
289   
290    # Form an XML response to the command
291    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
292    $xml_response .= "<GTIResponse>\n";
293    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
294   
295    foreach my $translation_file (@$gti_translation_files) {   
296        my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
297        my $target_file_name = "";
298       
299        if ($translation_file->{'key'} eq "gs3interface") {
300            my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
301            &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
302           
303            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
304            my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
305           
306            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
307            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
308            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
309            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
310        }
311        else {
312            # Get (and check) the translation configuration
313            my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
314            $target_file_name = $target_file;
315           
316            # Parse the source language and target language files
317            my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
318            my @source_file_lines = &read_file_lines($source_file_path);
319            my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
320           
321            my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
322            my @target_file_lines = &read_file_lines($target_file_path);
323            my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
324           
325            # Filter out any automatically translated chunks
326            foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
327                if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
328                    delete $source_file_key_to_line_mapping{$chunk_key};
329                    delete $target_file_key_to_line_mapping{$chunk_key};
330                }
331            }
332           
333            my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
334            my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
335           
336            # Determine the target file chunks requiring translation
337            my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
338           
339            # Determine the target file chunks requiring updating
340            my @target_file_keys_requiring_updating = ();
341            if (-e $target_file_path) {
342                my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
343                my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
344                @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
345            }
346           
347            $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
348            $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
349            $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
350            $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
351        }
352       
353        &log_message("Status of " . $translation_file->{'key'});
354        &log_message("Number of source chunks: " . $num_source_chunks);
355        &log_message("Number of target chunks: " . $num_target_chunks);
356        &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
357        &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
358       
359        $xml_response .= "    <TranslationFile"
360        . " key=\"" . $translation_file->{'key'} . "\""
361        . " target_file_path=\"" . $target_file_name . "\""
362        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
363        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
364        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
365    }
366   
367    $xml_response .= "  </LanguageStatus>\n";
368   
369    $xml_response .= "</GTIResponse>\n";
370    return $xml_response;
371}
372
373
374sub search_chunks
375{
376    # The code of the target language (ensure it is lowercase)
377    my $target_language_code = lc(shift(@_));
378    # The key of the file to translate (ensure it is lowercase)
379    my $translation_file_key = lc(shift(@_));
380    # The query string
381    my $query_string = join(' ', @_);
382   
383    # Check that the necessary arguments were supplied
384    if (!$target_language_code || !$translation_file_key || !$query_string) {
385        &throw_fatal_error("Missing command argument.");
386    }
387   
388    my ($source_file, $target_file, $translation_file_type) = ();
389    my %source_file_key_to_text_mapping = ();
390    my %target_file_key_to_text_mapping = ();
391   
392   
393    if ($translation_file_key ne "gs3interface") {
394        # Get (and check) the translation configuration
395        ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
396       
397        # Parse the source language and target language files
398        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
399        my @source_file_lines = &read_file_lines($source_file_path);
400        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
401       
402        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
403        my @target_file_lines = &read_file_lines($target_file_path);
404        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
405       
406        # Filter out any automatically translated chunks
407        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
408            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
409                delete $source_file_key_to_line_mapping{$chunk_key};
410                delete $target_file_key_to_line_mapping{$chunk_key};
411            }
412        }
413       
414        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
415        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
416    }
417    else {
418        # Not needed in this case
419        my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
420        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
421        \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
422    }
423   
424    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
425    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
426   
427    # Determine the target file chunks matching the query
428    my @target_file_keys_matching_query = ();
429    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
430        my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
431        if ($target_file_text =~ /$query_string/i) {
432            # &log_message("Chunk with key $chunk_key matches query.");
433            push(@target_file_keys_matching_query, $chunk_key);
434        }
435    }
436   
437    # Form an XML response to the command
438    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
439    $xml_response .= "<GTIResponse>\n";
440   
441    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
442    foreach my $chunk_key (@target_file_keys_matching_query) {
443        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
444       
445        $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
446        $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
447        $xml_response .= "    </Chunk>\n";
448    }
449    $xml_response .= "  </ChunksMatchingQuery>\n";
450   
451    $xml_response .= "</GTIResponse>\n";
452    return $xml_response;
453}
454
455
456sub submit_translations
457{
458    # The code of the target language (ensure it is lowercase)
459    my $target_language_code = lc(shift(@_));
460    # The key of the file to translate (ensure it is lowercase)
461    my $translation_file_key = lc(shift(@_));
462    # The username of the translation submitter
463    my $submitter_username = shift(@_);
464    # Whether to submit a target chunk even if it hasn't changed
465    my $force_submission_flag = shift(@_);
466   
467    # Check that the necessary arguments were supplied
468    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
469        &log_message("Fatal error (but cannot be thrown): Missing command argument.");
470        die "\n";
471    }
472   
473    my %source_file_key_to_text_mapping = ();
474    my %source_file_key_to_gti_comment_mapping = ();
475    my %target_file_key_to_text_mapping = ();
476    my %target_file_key_to_gti_comment_mapping = ();
477   
478    my (@source_file_lines, @target_file_lines) = ();
479    my ($source_file, $target_file, $translation_file_type);
480   
481   
482    if ($translation_file_key ne "gs3interface") {
483        # Get (and check) the translation configuration
484        ($source_file, $target_file, $translation_file_type)
485        = &get_translation_configuration($target_language_code, $translation_file_key);
486       
487        # Parse the source language and target language files
488        @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
489        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
490        %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
491        %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
492       
493        @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
494        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
495        %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
496        %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
497    }
498    else {
499        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
500        \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
501    }
502    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
503    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
504   
505    # Submission date
506    my $day = (localtime)[3];
507    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
508    my $year = (localtime)[5] + 1900;
509    my $submission_date = "$day-$month-$year";
510   
511    open(SUBMISSION, "-");
512    my @submission_lines = <SUBMISSION>;
513    close(SUBMISSION);
514   
515    # Remove any nasty carriage returns
516    # &log_message("Submission:");
517    foreach my $submission_line (@submission_lines) {
518        $submission_line =~ s/\r$//;
519        #&log_message("  $submission_line");
520    }
521   
522    my %source_file_key_to_submission_mapping = ();
523    my %target_file_key_to_submission_mapping = ();
524    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
525        # Read source file part of submission
526        if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
527            my $chunk_key = $1;
528           
529            # Read the source file text
530            my $source_file_chunk_text = "";
531            $i++;
532            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
533                $source_file_chunk_text .= $submission_lines[$i];
534                $i++;
535            }
536            $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
537            $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
538           
539            #&log_message("Source file key: $chunk_key");
540            #&log_message("Source file text: $source_file_chunk_text");
541            $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
542        }
543       
544        # Read target file part of submission
545        if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
546            my $chunk_key = $1;
547           
548            # Read the target file text
549            my $target_file_chunk_text = "";
550            $i++;
551            while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
552                $target_file_chunk_text .= $submission_lines[$i];
553                $i++;
554            }
555            $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
556            $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
557           
558            #&log_message("Target file key: $chunk_key");
559            #&log_message("Target file text: $target_file_chunk_text");
560            $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
561        }
562    }
563   
564    # -----------------------------------------
565    #   Validate the translation submissions
566    # -----------------------------------------
567   
568    # Check that the translations are valid
569    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
570        # Make sure the submitted chunk still exists in the source file
571        if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
572            &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
573            delete $source_file_key_to_submission_mapping{$chunk_key};
574            delete $target_file_key_to_submission_mapping{$chunk_key};
575            next;
576        }
577       
578        # Make sure the submitted source chunk matches the source file chunk
579        if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
580            &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
581            &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
582            &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
583            delete $source_file_key_to_submission_mapping{$chunk_key};
584            delete $target_file_key_to_submission_mapping{$chunk_key};
585            next;
586        }
587    }
588   
589    # Apply the submitted translations
590    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
591        # Only apply the submission if it is a change, unless -force_submission has been specified
592        if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
593            $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
594            $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
595        }
596    }
597   
598    if ($translation_file_key ne "gs3interface") {
599        eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
600    } else {
601        eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
602    }
603}
604
605
606sub create_glihelp_zip_file
607{
608    my $target_language_code = shift(@_);
609    my $translation_file_key = "glihelp";
610   
611    &log_message("Creating GLI Help zip file for $target_language_code");
612   
613    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
614   
615    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
616    if ( ! -e $classpath) {
617        &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
618    }   
619   
620    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
621    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
622   
623    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
624    if ( ! -e $gen_many_html_xsl_filepath) {
625        &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
626    }
627   
628    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
629    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
630   
631    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
632    $target_file_directory = $target_file_directory."/";
633   
634    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
635   
636    my $perl_exec = &util::get_perl_exec();
637    my $java_exec = "java";
638    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
639        $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
640    }
641   
642    my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
643    my $response = `$cmd`;
644       
645    $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
646    $response = `$cmd`;
647   
648    my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
649    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
650    $response = `$cmd`;
651}
652
653
654sub get_translation_configuration
655{
656    # Get the code of the target language
657    my $target_language_code = shift(@_);
658    # Get the key of the file to translate
659    my $translation_file_key = shift(@_);
660   
661    # Read the translation data from the gti.cfg file
662    my ($source_file, $target_file, $translation_file_type) =
663    &get_translation_data_for($target_language_code, $translation_file_key);
664   
665    # Check that the file to translate is defined in the gti.cfg file
666    if (!$source_file || !$target_file || !$translation_file_type) {
667        &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
668    }
669   
670    # Check that the source file exists
671    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
672    if (!-e $source_file_path) {
673        &throw_fatal_error("Source file $source_file_path does not exist.");
674    }
675   
676    # Check that the source file is up to date
677    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
678    # unless ($translation_file_is_not_in_cvs) {
679    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
680    my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
681    if ($source_file_cvs_status =~ /^C /) {
682        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
683    }
684    if ($source_file_cvs_status =~ /^M /) {
685        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
686    }
687    # }
688   
689    return ($source_file, $target_file, $translation_file_type);
690}
691
692
693sub get_translation_data_for
694{
695    my ($target_language_code, $translation_file_key) = @_;
696   
697    foreach my $translation_file (@$gti_translation_files) {
698        # If this isn't the correct translation file, move onto the next one
699        next if ($translation_file_key ne $translation_file->{'key'});
700       
701        # Resolve the target language file
702        my $target_language_file = $translation_file->{'target_file'};
703        if ($target_language_file =~ /(\{.+\;.+\})/) {
704            my $unresolved_target_language_file_part = $1;
705           
706            # Check for a special case for the target language code
707            if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
708                my $resolved_target_language_file_part = $2;
709                $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
710            }
711            # Otherwise use the last part as the default value
712            else {
713                my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
714            $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
715        }
716    }
717   
718    # Resolve instances of {iso_639_1_target_language_name}
719    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
720    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
721    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
722   
723    # Resolve instances of {target_language_code}
724    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
725   
726    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
727}
728
729return ();
730}
731
732
733sub read_file_lines
734{
735    my ($file_path) = @_;
736   
737    if (!open(FILE_IN, "<$file_path")) {
738        &log_message("Note: Could not open file $file_path.");
739        return ();
740    }
741    my @file_lines = <FILE_IN>;
742    close(FILE_IN);
743   
744    return @file_lines;
745}
746
747
748sub build_key_to_line_mapping
749{
750    my ($file_lines, $translation_file_type) = @_;
751    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
752}
753
754
755sub build_key_to_text_mapping
756{
757    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
758   
759    my %key_to_text_mapping = ();
760    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
761        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
762        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
763       
764        my $chunk_text = @$file_lines[$chunk_starting_line];
765        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
766            $chunk_text .= @$file_lines[$l];
767        }
768       
769        # Map from chunk key to text
770        eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
771    }
772   
773    return %key_to_text_mapping;
774}
775
776
777sub build_key_to_last_update_date_mapping
778{
779    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
780   
781    # If the files aren't in CVS then we can't tell anything about what needs updating
782    # return () if ($translation_file_is_not_in_cvs);
783   
784    # Build a mapping from key to CVS date
785    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
786    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
787   
788    # Build a mapping from key to comment date
789    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
790   
791    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
792    my %key_to_last_update_date_mapping = ();
793    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
794        # Use the CVS date as a starting point
795        my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
796        $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
797       
798        # If a comment date exists and it is after the CVS date, use that instead
799        # need to convert the comment date format to SVN format
800        my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
801        if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
802            my $chunk_comment_date = $1;           
803            if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
804                $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;         
805            }
806        }
807    }
808   
809    return %key_to_last_update_date_mapping;
810}
811
812
813sub build_key_to_cvs_date_mapping
814{
815    my ($filename, $translation_file_type) = @_;
816   
817    # Use SVN to annotate each line of the file with the date it was last edited
818    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
819    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
820   
821    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
822   
823    my @cvs_annotated_file_lines_date = ();
824    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
825        # Extract the date from the SVN annotation at the front
826        # svn format : 2007-07-16
827        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
828       
829        push(@cvs_annotated_file_lines_date, $1);
830       
831        # trim extra date information in svn annotation format
832        # 15:42:49 +1200 (Wed, 21 Jun 2006)
833        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
834    }   
835   
836    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
837    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
838   
839    my %key_to_cvs_date_mapping = ();
840    foreach my $chunk_key (keys(%key_to_line_mapping)) {
841        my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
842        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
843       
844        # Find the date this chunk was last edited, from the CVS annotation
845        my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
846        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
847            if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
848                # This part of the chunk has been updated more recently
849                $chunk_date = $cvs_annotated_file_lines_date[$l];
850               
851            }
852        }
853       
854        # Map from chunk key to CVS date
855        $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
856    }
857   
858    return %key_to_cvs_date_mapping;
859}
860
861
862sub build_key_to_gti_comment_mapping
863{
864    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
865   
866    my %key_to_gti_comment_mapping = ();
867    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
868        my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
869        my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
870       
871        my $chunk_text = @$file_lines[$chunk_starting_line];
872        for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
873            $chunk_text .= @$file_lines[$l];
874        }
875       
876        # Map from chunk key to GTI comment
877        my $chunk_gti_comment;
878        eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
879        $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
880    }
881   
882    return %key_to_gti_comment_mapping;
883}
884
885
886sub determine_chunks_requiring_translation
887{
888    my $source_file_key_to_text_mapping = shift(@_);
889    my $target_file_key_to_text_mapping = shift(@_);
890   
891    # Chunks needing translation are those in the source file with no translation in the target file
892    my @target_file_keys_requiring_translation = ();
893    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
894        if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
895            # &log_message("Chunk with key $chunk_key needs translating.");
896            push(@target_file_keys_requiring_translation, $chunk_key);
897        }
898    }
899   
900    return @target_file_keys_requiring_translation;
901}
902
903
904sub determine_chunks_requiring_updating
905{
906    my $source_file_key_to_last_update_date_mapping = shift(@_);
907    my $target_file_key_to_last_update_date_mapping = shift(@_);
908   
909    # Chunks needing updating are those in the target file that have been more recently edited in the source file
910    my @target_file_keys_requiring_updating = ();
911    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
912        my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
913        my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
914       
915        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
916       
917        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
918            # &log_message("Chunk with key $chunk_key needs updating.");
919            push(@target_file_keys_requiring_updating, $chunk_key);
920        }
921    }
922   
923    return @target_file_keys_requiring_updating;
924}
925
926
927sub is_chunk_automatically_translated
928{
929    my ($chunk_key, $translation_file_type) = @_;
930    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
931}
932
933
934sub make_text_xml_safe
935{
936    my $text = shift(@_);
937    $text =~ s/\&/\&amp\;/g;
938    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
939    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
940    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
941    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
942    $text =~ s/</\&lt\;/g;
943    $text =~ s/>/\&gt\;/g;
944    return $text;
945}
946
947
948sub unmake_text_xml_safe
949{
950    my $text = shift(@_);
951    $text =~ s/\&lt\;/</g;
952    $text =~ s/\&gt\;/>/g;
953    $text =~ s/\&amp\;/\&/g;
954    return $text;
955}
956
957
958# Returns 1 if $date1 is after $date2, 0 otherwise
959sub is_date_after_cvs
960{
961    my ($date1, $date2) = @_;
962    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
963    "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
964   
965    if(!defined $date1) {
966        return 1;
967    }
968   
969    my @date1parts = split(/-/, $date1);
970    my @date2parts = split(/-/, $date2);
971   
972    # Compare year - nasty because we have rolled over into a new century
973    my $year1 = $date1parts[2];
974    if ($year1 < 80) {
975        $year1 += 2000;
976    }
977    my $year2 = $date2parts[2];
978    if ($year2 < 80) {
979        $year2 += 2000;
980    }
981   
982    # Compare year
983    if ($year1 > $year2) {
984        return 1;
985    }
986    elsif ($year1 == $year2) {
987        # Year is the same, so compare month
988        if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
989            return 1;
990        }
991        elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
992            # Month is the same, so compare day
993            if ($date1parts[0] > $date2parts[0]) {
994                return 1;
995            }
996        }
997    }
998   
999    return 0;
1000}
1001
1002sub is_date_after
1003{
1004    my ($date1, $date2) = @_;
1005   
1006    if(!defined $date1) {
1007        return 1;
1008    }
1009    if(!defined $date2) {
1010        return 0;
1011    }
1012   
1013    # 16-Aug-2006
1014    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1015        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1016        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1017        $date1=$3 . "-" . $months{$2} . "-" . $1;
1018        # print "** converted date1: $date1\n";
1019    }
1020    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1021        my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1022        "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1023        $date2=$3 . "-" . $months{$2} . "-" . $1;
1024        # print "** converted date2: $date2\n";
1025    }
1026   
1027   
1028    # 2006-08-16
1029    my @date1parts = split(/-/, $date1);
1030    my @date2parts = split(/-/, $date2);
1031   
1032    # Compare year
1033    if ($date1parts[0] > $date2parts[0]) {
1034        return 1;
1035    }
1036    elsif ($date1parts[0] == $date2parts[0]) {
1037        # Year is the same, so compare month
1038        if ($date1parts[1] > $date2parts[1]) {
1039            return 1;
1040        }
1041        elsif ($date1parts[1] == $date2parts[1]) {
1042            # Month is the same, so compare day
1043            if ($date1parts[2] > $date2parts[2]) {
1044                return 1;
1045            }
1046        }
1047    }   
1048   
1049    return 0;
1050}
1051
1052
1053sub create_xml_response_for_chunks_requiring_work
1054{
1055    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1056   
1057    # Form an XML response to the command
1058    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1059    $xml_response .= "<GTIResponse>\n";
1060    $xml_response .= "  <TranslationFile"
1061    . " key=\"" . $translation_file_key . "\""
1062    . " target_file_path=\"" . $target_file . "\""
1063    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1064    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1065    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1066   
1067    # Do chunks requiring translation first
1068    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1069        $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1070    }
1071    else {
1072        $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1073    }
1074   
1075    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1076    foreach my $chunk_key (@sorted_chunk_keys) {
1077        last if ($num_chunks_to_return == 0);
1078       
1079        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1080        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1081       
1082        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1083        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1084        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1085        $xml_response .= "    </Chunk>\n";
1086       
1087        $num_chunks_to_return--;
1088    }
1089   
1090    $xml_response .= "  </ChunksRequiringTranslation>\n";
1091   
1092    # Then do chunks requiring updating
1093    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1094        $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1095    }
1096    else {
1097        $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1098    }
1099   
1100    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1101    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1102    foreach my $chunk_key (@sorted_chunk_keys) {
1103        last if ($num_chunks_to_return == 0);
1104       
1105        my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1106        my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1107        my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1108        my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1109       
1110        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1111        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1112        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1113        $xml_response .= "    </Chunk>\n";
1114       
1115        $num_chunks_to_return--;
1116    }
1117   
1118    $xml_response .= "  </ChunksRequiringUpdating>\n";
1119   
1120    $xml_response .= "</GTIResponse>\n";
1121   
1122    return $xml_response;
1123}
1124
1125
1126sub create_xml_response_for_all_chunks
1127{
1128    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1129   
1130    # Form an XML response to the command
1131    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1132    $xml_response .= "<GTIResponse>\n";
1133    $xml_response .= "  <TranslationFile"
1134    . " key=\"" . $translation_file_key . "\""
1135    . " target_file_path=\"" . $target_file . "\"\/>\n";
1136   
1137    # Do all the chunks
1138    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1139   
1140    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1141    foreach my $chunk_key (@sorted_chunk_keys) {
1142        my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1143        my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1144       
1145        $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1146        $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1147        if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1148            my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1149            my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1150            $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1151        }
1152        else {
1153            $xml_response .= "      <TargetFileText></TargetFileText>\n";
1154        }
1155       
1156        $xml_response .= "    </Chunk>\n";
1157    }
1158    $xml_response .= "  </Chunks>\n";
1159   
1160    $xml_response .= "</GTIResponse>\n";
1161    return $xml_response;
1162}
1163
1164
1165
1166# ==========================================================================================
1167#   MACROFILE FUNCTIONS
1168
1169sub build_key_to_line_mapping_for_macrofile
1170{
1171    my (@file_lines) = @_;
1172   
1173    my $macro_package;
1174    my %chunk_key_to_line_mapping = ();
1175    # Process the contents of the file, line by line
1176    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1177        my $line = $file_lines[$i];
1178        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1179       
1180        # Check if a new package is being defined
1181        if ($line =~ m/^package\s+(.+)/) {
1182            $macro_package = $1;
1183        }
1184       
1185        # Line contains a macro name
1186        elsif ($line =~ m/^(_\w+_)/) {
1187            my $macro_key = $1;
1188            $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1189           
1190            # While there is still text of the macro to go...
1191            my $startline = $i;
1192            while ($line !~ /\}$/) {
1193                $i++;
1194                if ($i == scalar(@file_lines)) {
1195                    &throw_fatal_error("Could not find end of macro $macro_key.");
1196                }
1197                $line = $file_lines[$i];
1198                $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1199            }
1200       
1201        # The chunk key consists of the package name and the macro key
1202        my $chunk_key = $macro_package . "." . $macro_key;
1203        # Map from chunk key to line
1204        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1205    }
1206   
1207    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1208    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1209    # The chunk key consists of package name and macro key
1210    my $chunk_key = $macro_package . "." . $1;
1211    # Map from chunk key to line
1212    $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1213}
1214}
1215
1216return %chunk_key_to_line_mapping;
1217}
1218
1219
1220sub import_chunk_from_macrofile
1221{
1222    my ($chunk_text) = @_;
1223   
1224    # Is this an icon macro??
1225    if ($chunk_text =~ /^\#\# (.*)/) {
1226        # Extract image macro text
1227        $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1228        $chunk_text = $1;
1229   
1230    # Remove enclosing quotes
1231    $chunk_text =~ s/^\"//;
1232    $chunk_text =~ s/\"$//;
1233}
1234
1235# No, so it must be a text macro
1236else {
1237    # Remove macro key
1238    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1239   
1240    # Remove language specifier
1241    $chunk_text =~ s/^\[l=.*\](\s*)//;
1242   
1243    # Remove braces enclosing text
1244    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1245}
1246
1247return $chunk_text;
1248}
1249
1250
1251sub get_macrofile_chunk_gti_comment
1252{
1253    my ($chunk_text) = @_;
1254   
1255    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1256    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1257        return $1;
1258}
1259
1260return undef;
1261}
1262
1263
1264sub is_macrofile_chunk_automatically_translated
1265{
1266    my ($chunk_key) = @_;
1267   
1268    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1269    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1270        return 1;
1271    }
1272   
1273    return 0;
1274}
1275
1276
1277# Use the source file to generate a target file that is formatted the same
1278sub write_translated_macrofile
1279{
1280    my $source_file = shift(@_);  # Not used
1281    my @source_file_lines = @{shift(@_)};
1282    my $source_file_key_to_text_mapping = shift(@_);
1283    my $target_file = shift(@_);
1284    my @target_file_lines = @{shift(@_)};
1285    my $target_file_key_to_text_mapping = shift(@_);
1286    my $target_file_key_to_gti_comment_mapping = shift(@_);
1287    my $target_language_code = shift(@_);
1288   
1289    # Build a mapping from source file line to chunk key
1290    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1291    my %source_file_line_to_key_mapping = ();
1292    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1293        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1294    }
1295    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1296    my $source_file_line_number = 0;
1297   
1298    # Build a mapping from target file line to chunk key
1299    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1300    my %target_file_line_to_key_mapping = ();
1301    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1302        $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1303    }
1304    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1305   
1306    # Write the new target file
1307    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1308    if (!open(TARGET_FILE, ">$target_file_path")) {
1309        &throw_fatal_error("Could not write target file $target_file_path.");
1310    }
1311   
1312    # Use the header from the target file, to keep language and author information
1313    if (scalar(@target_file_line_keys) > 0) {
1314        my $target_file_line_number = 0;
1315        my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1316        while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1317            my $target_file_line = $target_file_lines[$target_file_line_number];
1318            last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1319                print TARGET_FILE $target_file_line;
1320            $target_file_line_number++;
1321        }
1322       
1323        $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1324    }
1325   
1326    # Model the new target file on the source file, with the target file translations
1327    foreach my $line_key (@source_file_line_keys) {
1328        # Fill in the gaps before this chunk starts
1329        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1330        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1331        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1332            print TARGET_FILE $source_file_lines[$source_file_line_number];
1333            $source_file_line_number++;
1334        }
1335        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1336       
1337        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1338        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1339        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1340       
1341        my $macrofile_key = $chunk_key;
1342        $macrofile_key =~ s/^(.+?)\.//;
1343       
1344        # If no translation exists for this chunk, show this, and move on
1345        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1346            print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1347            next;
1348        }
1349       
1350        # Grab the source chunk text
1351        my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1352        for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1353            $source_file_chunk .= $source_file_lines[$l];
1354        }
1355       
1356        # Is this an icon macro??
1357        if ($source_file_chunk =~ /^\#\# (.*)/) {
1358            # Escape any newline and question mark characters so the source text is replaced correctly
1359            $source_file_chunk_text =~ s/\\/\\\\/g;
1360        $source_file_chunk_text =~ s/\?/\\\?/g;
1361       
1362        # Build the new target chunk from the source chunk
1363        my $target_file_chunk = $source_file_chunk;
1364        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1365        $target_file_chunk =~ s/(\s)*$//;
1366        print TARGET_FILE "$target_file_chunk";
1367    }
1368   
1369    # No, it is just a normal text macro
1370    else {
1371        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1372    }
1373   
1374    # Add the "updated" comment, if one exists
1375    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1376        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1377    }
1378    print TARGET_FILE "\n";
1379}
1380
1381close(TARGET_FILE);
1382}
1383
1384
1385sub sort_by_line
1386{
1387    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1388}
1389
1390
1391# ==========================================================================================
1392#   RESOURCE BUNDLE FUNCTIONS
1393
1394sub build_key_to_line_mapping_for_resource_bundle
1395{
1396    my (@file_lines) = @_;
1397   
1398    my %chunk_key_to_line_mapping = ();
1399    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1400        my $line = $file_lines[$i];
1401        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1402       
1403        # Line contains a dictionary string
1404        if ($line =~ /^(\S+?)[:|=](.*)$/) {
1405            my $chunk_key = $1;
1406           
1407            # Map from chunk key to line
1408            $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1409        }
1410    }
1411   
1412    return %chunk_key_to_line_mapping;
1413}
1414
1415
1416sub import_chunk_from_resource_bundle
1417{
1418    my ($chunk_text) = @_;
1419   
1420    # Simple: just remove string key
1421    $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1422    $chunk_text =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1423    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1424   
1425    return $chunk_text;
1426}
1427
1428
1429sub get_resource_bundle_chunk_gti_comment
1430{
1431    my ($chunk_text) = @_;
1432   
1433    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1434    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1435        return $1;
1436}
1437
1438return undef;
1439}
1440
1441
1442sub is_resource_bundle_chunk_automatically_translated
1443{
1444    # No resource bundle chunks are automatically translated
1445    return 0;
1446}
1447
1448
1449sub write_translated_resource_bundle
1450{
1451    my $source_file = shift(@_);  # Not used
1452    my @source_file_lines = @{shift(@_)};
1453    my $source_file_key_to_text_mapping = shift(@_);
1454    my $target_file = shift(@_);
1455    my @target_file_lines = @{shift(@_)};  # Not used
1456    my $target_file_key_to_text_mapping = shift(@_);
1457    my $target_file_key_to_gti_comment_mapping = shift(@_);
1458    my $target_language_code = shift(@_);  # Not used
1459   
1460    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1461    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1462    my %source_file_line_to_key_mapping = ();
1463    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1464        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1465    }
1466   
1467    # Write the new target file
1468    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1469    if (!open(TARGET_FILE, ">$target_file_path")) {
1470        &throw_fatal_error("Could not write target file $target_file_path.");
1471    }
1472   
1473    # Model the new target file on the source file, with the target file translations
1474    my $source_file_line_number = 0;
1475    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1476        # Fill in the gaps before this chunk starts
1477        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1478        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1479        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1480            print TARGET_FILE $source_file_lines[$source_file_line_number];
1481            $source_file_line_number++;
1482        }
1483        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1484       
1485        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1486        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1487        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1488       
1489        # If no translation exists for this chunk, show this, and move on
1490        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1491            print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1492            next;
1493        }
1494       
1495        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1496        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1497            print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1498        }
1499        print TARGET_FILE "\n";
1500    }
1501   
1502    close(TARGET_FILE);
1503}
1504
1505
1506# ==========================================================================================
1507#   GREENSTONE XML FUNCTIONS
1508
1509sub build_key_to_line_mapping_for_greenstone_xml
1510{
1511    my (@file_lines) = @_;
1512   
1513    my %chunk_key_to_line_mapping = ();
1514    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1515        my $line = $file_lines[$i];
1516        $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1517       
1518        # Line contains a string to translate
1519        if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1520            my $chunk_key = $1;
1521            $line =~ s/\s*$//;  # Remove any nasty whitespace
1522            $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1523           
1524            # While there is still text of the string to go...
1525            my $startline = $i;
1526            while ($line !~ /<\/Text>$/) {
1527                $i++;
1528                if ($i == scalar(@file_lines)) {
1529                    &throw_fatal_error("Could not find end of string $chunk_key.");
1530                }
1531                $line = $file_lines[$i];
1532                $line =~ s/\s*$//;  # Remove any nasty whitespace
1533                $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1534            }
1535           
1536            # Map from chunk key to line
1537            if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1538                $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1539            }
1540            else {
1541                &throw_fatal_error("Duplicate key $chunk_key.");
1542            }
1543        }
1544    }
1545   
1546    return %chunk_key_to_line_mapping;
1547}
1548
1549
1550sub import_chunk_from_greenstone_xml
1551{
1552    my ($chunk_text) = @_;
1553   
1554    # Simple: just remove the Text tags
1555    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1556    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1557    $chunk_text =~ s/<\/Text>$//;
1558   
1559    return $chunk_text;
1560}
1561
1562
1563sub get_greenstone_xml_chunk_gti_comment
1564{
1565    my ($chunk_text) = @_;
1566   
1567    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1568    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1569        return $1;
1570    }
1571   
1572    return undef;
1573}
1574
1575
1576sub is_greenstone_xml_chunk_automatically_translated
1577{
1578    # No greenstone XML chunks are automatically translated
1579    return 0;
1580}
1581
1582
1583sub write_translated_greenstone_xml
1584{
1585    my $source_file = shift(@_);  # Not used
1586    my @source_file_lines = @{shift(@_)};
1587    my $source_file_key_to_text_mapping = shift(@_);
1588    my $target_file = shift(@_);
1589    my @target_file_lines = @{shift(@_)};  # Not used
1590    my $target_file_key_to_text_mapping = shift(@_);
1591    my $target_file_key_to_gti_comment_mapping = shift(@_);
1592    my $target_language_code = shift(@_);  # Not used
1593   
1594    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1595    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1596    my %source_file_line_to_key_mapping = ();
1597    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1598        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1599    }
1600   
1601    # Write the new target file
1602    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1603    if (!open(TARGET_FILE, ">$target_file_path")) {
1604        &throw_fatal_error("Could not write target file $target_file_path.");
1605    }
1606   
1607    # Model the new target file on the source file, with the target file translations
1608    my $source_file_line_number = 0;
1609    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1610        # Fill in the gaps before this chunk starts
1611        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1612        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1613        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1614            print TARGET_FILE $source_file_lines[$source_file_line_number];
1615            $source_file_line_number++;
1616        }
1617        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1618       
1619        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1620        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1621        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1622        $target_file_chunk_text =~ s/(\n)*$//g;
1623       
1624        # If no translation exists for this chunk, show this, and move on
1625        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1626            print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1627            next;
1628        }
1629       
1630        print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1631        if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1632            my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1633            $chunk_gti_comment =~ s/^Updated //;
1634            print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1635        }
1636        print TARGET_FILE "\n";
1637    }
1638   
1639    # Fill in the end of the file
1640    while ($source_file_line_number < scalar(@source_file_lines)) {
1641        print TARGET_FILE $source_file_lines[$source_file_line_number];
1642        $source_file_line_number++;
1643    }
1644   
1645    close(TARGET_FILE);
1646}
1647
1648
1649# ==========================================================================================
1650#   GREENSTONE3 FUNCTIONS
1651
1652sub get_all_chunks_for_gs3
1653{
1654    # The code of the target language (ensure it is lowercase)
1655    my $target_language_code = lc(shift(@_));
1656    my $translation_file_key = lc(shift(@_));
1657   
1658    # Check that the necessary arguments were supplied
1659    if (!$target_language_code) {
1660        &throw_fatal_error("Missing command argument.");
1661    }
1662   
1663    # Get (and check) the translation configuration
1664    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1665   
1666    my %source_files_key_to_text_mapping = ();
1667    my %target_files_key_to_text_mapping = ();
1668    my %source_files_key_to_last_update_date_mapping = ();
1669    my %target_files_key_to_last_update_date_mapping = ();
1670   
1671    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1672   
1673    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1674    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1675   
1676    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1677    return $xml_response;
1678}
1679
1680
1681sub get_first_n_chunks_requiring_work_for_gs3
1682{
1683    # The code of the target language (ensure it is lowercase)
1684    my $target_language_code = lc(shift(@_));
1685    # The key of the file to translate (ensure it is lowercase)
1686    my $translation_file_key = lc(shift(@_));
1687    # The number of chunks to return (defaults to one if not specified)
1688    my $num_chunks_to_return = shift(@_) || "1";
1689   
1690    # Check that the necessary arguments were supplied
1691    if (!$target_language_code || !$translation_file_key) {
1692        &throw_fatal_error("Missing command argument.");
1693    }
1694   
1695    my %source_files_key_to_text_mapping = ();
1696    my %target_files_key_to_text_mapping = ();
1697    my %source_files_key_to_last_update_date_mapping = ();
1698    my %target_files_key_to_last_update_date_mapping = ();
1699   
1700    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1701    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1702   
1703    # Determine the target file chunks requiring translation
1704    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1705    # Determine the target file chunks requiring updating
1706    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1707    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1708    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1709   
1710    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1711    \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1712    $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1713    \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1714   
1715    return $xml_response;
1716}
1717
1718
1719
1720sub build_gs3_configuration
1721{
1722    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1723    $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1724   
1725    my $source_file_directory = "greenstone3";
1726    my $translation_file_type = "resource_bundle";
1727   
1728    foreach my $interface_file_key (@gs3_interface_files) {
1729       
1730        &log_message("Greenstone 3 interface file: " . $interface_file_key);
1731       
1732        # Parse the source language and target language files
1733        my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1734        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1735        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1736        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1737        my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1738       
1739        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1740        my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1741        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1742        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1743        my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1744       
1745       
1746        # Filter out any automatically translated chunks
1747        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1748            if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1749                delete $source_file_key_to_line_mapping{$chunk_key};
1750                delete $target_file_key_to_line_mapping{$chunk_key};
1751            }
1752        }
1753       
1754        &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1755        &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1756       
1757        foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1758            my $global_chunk_key = "$interface_file_key.$chunk_key";
1759            $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1760            $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1761           
1762            if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1763                $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1764                $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1765            }
1766        }   
1767    }
1768}
1769
1770
1771sub write_translated_gs3interface
1772{
1773    my $source_file_key_to_text_mapping = shift(@_);
1774    my $target_file_key_to_text_mapping = shift(@_);
1775    my $target_file_key_to_gti_comment_mapping = shift(@_);
1776    my $target_language_code = shift(@_);
1777   
1778    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1779   
1780    my %translated_interface_file_keys = ();
1781    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1782        $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1783        if (!defined $translated_interface_file_keys{$1}) {
1784            &log_message("Updated interface file: " . $1); 
1785            $translated_interface_file_keys{$1}="";
1786        }
1787    }
1788    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1789   
1790    my $source_file_directory = "greenstone3";   
1791   
1792    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1793       
1794        # Build a mapping from chunk key to source file line, and from source file line to chunk key
1795        my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1796        my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1797        my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1798        my %source_file_line_to_key_mapping = ();
1799        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1800            $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1801        }
1802       
1803        # Write the new target file
1804        my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1805        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1806        if (!open(TARGET_FILE, ">$target_file_path")) {
1807            &throw_fatal_error("Could not write target file $target_file_path.");
1808        }
1809       
1810        # Model the new target file on the source file, with the target file translations
1811        my $source_file_line_number = 0;
1812        foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1813            # Fill in the gaps before this chunk starts
1814            my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1815            my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1816            while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1817                print TARGET_FILE $source_file_lines[$source_file_line_number];
1818                $source_file_line_number++;
1819            }
1820            $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1821           
1822            my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1823            my $global_chunk_key = "$interface_file_key.$chunk_key";
1824            my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1825            my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1826           
1827            # If no translation exists for this chunk, show this, and move on
1828            if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1829                print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1830                next;
1831            }
1832           
1833            print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1834            if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1835                print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1836            }
1837            print TARGET_FILE "\n";
1838        }
1839       
1840        close(TARGET_FILE);
1841    }           
1842}
1843
1844&main(@ARGV);
Note: See TracBrowser for help on using the browser.