root/gsdl/trunk/bin/script/gti.pl @ 18460

Revision 18460, 76.7 KB (checked in by anna, 10 years ago)

Added several functions in GTI.\n 1. Create spreadsheet for all the strings in current module (gtiaction.cpp, english2.dm).\n 2. Create operational files for GLI Help (HTML files etc) as requested, zip them and provide for download (gtiaction.h|cpp, gti.dm, gti.pl).\n 3. Functions for processing GS3 interfaces (gti.pl).

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39
40
41#my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
42#my $anonymous_svn_root = "http://http://svn.greenstone.org/gsdl/trunk/";
43my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
44my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
45my $source_language_code = "en";  # This is non-negiotable
46
47my $gti_translation_files =
48    [ # Greenstone macrofiles
49      { 'key' => "coredm",
50    'file_type' => "macrofile",
51    'source_file' => "macros/english.dm",
52    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
53      { 'key' => "auxdm",
54    'file_type' => "macrofile",
55    'source_file' => "macros/english2.dm",
56    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
57
58      # GLI dictionary
59      { 'key' => "glidict",
60    'file_type' => "resource_bundle",
61    'source_file' => "gli/classes/dictionary.properties",
62    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
63
64      # GLI help
65      { 'key' => "glihelp",
66    'file_type' => "greenstone_xml",
67    'source_file' => "gli/help/en/help.xml",
68    'target_file' => "gli/help/{target_language_code}/help.xml" },
69
70      # Greenstone Perl modules
71      { 'key' => "perlmodules",
72    'file_type' => "resource_bundle",
73    'source_file' => "perllib/strings.properties",
74    'target_file' => "perllib/strings_{target_language_code}.properties" },
75
76      # Greenstone tutorial exercises
77      # { 'key' => "tutorials",
78    # 'file_type' => "greenstone_xml",
79    # 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
80    # 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
81
82      # new Greenstone.org
83      { 'key' => "greenorg",
84        'file_type' => "resource_bundle",
85        'source_file' => "greenstoneorg/website/classes/Gsc.properties",
86        'target_file' => "greenstoneorg/website/classes/Gsc_{iso_639_1_target_language_name}.properties"
87    # 'file_type' => "macrofile",
88    # 'source_file' => "greenorg/macros/english.dm",
89    # 'target_file' => "greenorg/macros/{iso_639_1_target_language_name}.dm"
90      }
91
92      # { 'key' => "gs3interface",
93    #'file_type' => "resource_bundle",
94    #'source_file' => "greenstone3",
95    #'target_file' => "greenstone3"
96      #}
97    ];
98
99my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
100
101
102sub main
103{
104    # Get the command to process, and any arguments
105    my $gti_command = shift(@_);
106    my @gti_command_arguments = @_;
107    my $module = $_[1];
108
109    # Open the GTI log file for appending, or write to STDERR if that fails
110    if (!open(GTI_LOG, ">>$gti_log_file")) {
111    open(GTI_LOG, ">&STDERR");
112    }
113
114    # Log the command that launched this script
115    &log_message("Command: $0 @ARGV");
116
117    # Check that a command was supplied
118    if (!$gti_command) {
119    &throw_fatal_error("Missing command.");
120    }     
121
122    # Process the command
123    if ($gti_command =~ /^get-all-chunks$/i) {
124    # Check that GS3 interface is the target
125    if ($module eq "gs3interface") {       
126        print &get_all_chunks_for_gs3(@gti_command_arguments);
127    } else {
128        print &get_all_chunks(@gti_command_arguments);
129    }
130    }
131    if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
132    if ($module eq "gs3interface") {       
133        print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
134    } else {
135        print &get_first_n_chunks_requiring_work(@gti_command_arguments);
136    }
137    }
138    if ($gti_command =~ /^get-language-status$/i) {
139    print &get_language_status(@gti_command_arguments);       
140    }
141    if ($gti_command =~ /^search-chunks$/i) {
142    print &search_chunks(@gti_command_arguments);
143    }
144    if ($gti_command =~ /^submit-translations$/i) {
145    # This command cannot produce any output since it reads input
146    &submit_translations(@gti_command_arguments);
147    }
148    if ($gti_command =~ /^create-glihelp-zip-file$/i) {
149    # This command cannot produce any output since it reads input
150    &create_glihelp_zip_file(@gti_command_arguments);
151    }
152#    else {
153    # The command was not recognized
154    #&throw_fatal_error("Unknown command \"$gti_command\".");
155    #}
156}
157
158
159sub throw_fatal_error
160{
161    my $error_message = shift(@_);
162
163    # Write an XML error response
164    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
165    print "<GTIResponse>\n";
166    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
167    print "</GTIResponse>\n";
168
169    # Log the error message, then die
170    &log_message("Error: $error_message");
171    die "\n";
172}
173
174
175sub log_message
176{
177    my $log_message = shift(@_);
178    print GTI_LOG time() . " -- " . $log_message . "\n";
179}
180
181
182sub get_all_chunks
183{
184    # The code of the target language (ensure it is lowercase)
185    my $target_language_code = lc(shift(@_));
186    # The key of the file to translate (ensure it is lowercase)
187    my $translation_file_key = lc(shift(@_));
188
189    # Check that the necessary arguments were supplied
190    if (!$target_language_code || !$translation_file_key) {
191    &throw_fatal_error("Missing command argument.");
192    }
193
194    # Get (and check) the translation configuration
195    my ($source_file, $target_file, $translation_file_type)
196    = &get_translation_configuration($target_language_code, $translation_file_key);
197
198    # Parse the source language and target language files
199    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
200    my @source_file_lines = &read_file_lines($source_file_path);
201    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
202   
203    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
204    my @target_file_lines = &read_file_lines($target_file_path);
205    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
206
207    # Filter out any automatically translated chunks
208    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
209    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
210        delete $source_file_key_to_line_mapping{$chunk_key};
211        delete $target_file_key_to_line_mapping{$chunk_key};
212    }
213    }
214
215    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
216    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
217    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
218    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
219
220    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
221    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
222
223    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
224   
225    return $xml_response;
226}
227
228
229sub get_first_n_chunks_requiring_work
230{
231    # The code of the target language (ensure it is lowercase)
232    my $target_language_code = lc(shift(@_));
233    # The key of the file to translate (ensure it is lowercase)
234    my $translation_file_key = lc(shift(@_));
235    # The number of chunks to return (defaults to one if not specified)
236    my $num_chunks_to_return = shift(@_) || "1";
237
238    # Check that the necessary arguments were supplied
239    if (!$target_language_code || !$translation_file_key) {
240    &throw_fatal_error("Missing command argument.");
241    }
242
243    # Get (and check) the translation configuration
244    my ($source_file, $target_file, $translation_file_type)
245    = &get_translation_configuration($target_language_code, $translation_file_key);
246
247    # Parse the source language and target language files
248    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
249    my @source_file_lines = &read_file_lines($source_file_path);
250    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
251   
252    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
253    my @target_file_lines = &read_file_lines($target_file_path);
254    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
255
256    # Filter out any automatically translated chunks
257    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
258    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
259        delete $source_file_key_to_line_mapping{$chunk_key};
260        delete $target_file_key_to_line_mapping{$chunk_key};
261    }
262    }
263
264    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
265    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
266    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
267    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
268
269    # Determine the target file chunks requiring translation
270    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
271    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
272
273    # Determine the target file chunks requiring updating
274    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
275    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
276    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
277    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
278
279    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
280   
281    return $xml_response;
282}
283
284
285sub get_language_status
286{
287    # The code of the target language (ensure it is lowercase)
288    my $target_language_code = lc(shift(@_));
289
290    # Check that the necessary arguments were supplied
291    if (!$target_language_code) {
292    &throw_fatal_error("Missing command argument.");
293    }
294
295    # Form an XML response to the command
296    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
297    $xml_response .= "<GTIResponse>\n";
298    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
299
300    foreach my $translation_file (@$gti_translation_files) {   
301    my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
302    my $target_file_name = "";
303   
304    if ($translation_file->{'key'} eq "gs3interface") {
305        my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
306        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
307       
308        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
309        my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
310
311        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
312        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
313        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
314        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
315    }
316    else {
317        # Get (and check) the translation configuration
318        my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
319        $target_file_name = $target_file;
320
321        # Parse the source language and target language files
322        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
323        my @source_file_lines = &read_file_lines($source_file_path);
324        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
325
326        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
327        my @target_file_lines = &read_file_lines($target_file_path);
328        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
329
330        # Filter out any automatically translated chunks
331        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
332        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
333            delete $source_file_key_to_line_mapping{$chunk_key};
334            delete $target_file_key_to_line_mapping{$chunk_key};
335        }
336        }
337
338        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
339        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
340
341        # Determine the target file chunks requiring translation
342        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
343
344        # Determine the target file chunks requiring updating
345        my @target_file_keys_requiring_updating = ();
346        if (-e $target_file_path) {
347        my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
348        my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
349        @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
350        }
351
352        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
353        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
354        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
355        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
356    }
357   
358    &log_message("Status of " . $translation_file->{'key'});
359    &log_message("Number of source chunks: " . $num_source_chunks);
360    &log_message("Number of target chunks: " . $num_target_chunks);
361    &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
362    &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
363
364    $xml_response .= "    <TranslationFile"
365        . " key=\"" . $translation_file->{'key'} . "\""
366        . " target_file_path=\"" . $target_file_name . "\""
367        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
368        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
369        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
370    }
371
372    $xml_response .= "  </LanguageStatus>\n";
373
374    $xml_response .= "</GTIResponse>\n";
375    return $xml_response;
376}
377
378
379sub search_chunks
380{
381    # The code of the target language (ensure it is lowercase)
382    my $target_language_code = lc(shift(@_));
383    # The key of the file to translate (ensure it is lowercase)
384    my $translation_file_key = lc(shift(@_));
385    # The query string
386    my $query_string = join(' ', @_);
387
388    # Check that the necessary arguments were supplied
389    if (!$target_language_code || !$translation_file_key || !$query_string) {
390    &throw_fatal_error("Missing command argument.");
391    }
392
393    my ($source_file, $target_file, $translation_file_type) = ();
394    my %source_file_key_to_text_mapping = ();
395    my %target_file_key_to_text_mapping = ();
396   
397   
398    if ($translation_file_key ne "gs3interface") {
399    # Get (and check) the translation configuration
400    ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
401
402    # Parse the source language and target language files
403    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
404    my @source_file_lines = &read_file_lines($source_file_path);
405    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
406
407    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
408    my @target_file_lines = &read_file_lines($target_file_path);
409    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
410
411    # Filter out any automatically translated chunks
412    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
413        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
414        delete $source_file_key_to_line_mapping{$chunk_key};
415        delete $target_file_key_to_line_mapping{$chunk_key};
416        }
417    }
418
419    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
420    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
421    }
422    else {
423    # Not needed in this case
424    my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
425    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
426                 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
427    }
428
429    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
430    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
431
432    # Determine the target file chunks matching the query
433    my @target_file_keys_matching_query = ();
434    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
435    my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
436    if ($target_file_text =~ /$query_string/i) {
437        # &log_message("Chunk with key $chunk_key matches query.");
438        push(@target_file_keys_matching_query, $chunk_key);
439    }
440    }
441
442    # Form an XML response to the command
443    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
444    $xml_response .= "<GTIResponse>\n";
445
446    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
447    foreach my $chunk_key (@target_file_keys_matching_query) {
448    my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
449
450    $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
451    $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
452    $xml_response .= "    </Chunk>\n";
453    }
454    $xml_response .= "  </ChunksMatchingQuery>\n";
455
456    $xml_response .= "</GTIResponse>\n";
457    return $xml_response;
458}
459
460
461sub submit_translations
462{
463    # The code of the target language (ensure it is lowercase)
464    my $target_language_code = lc(shift(@_));
465    # The key of the file to translate (ensure it is lowercase)
466    my $translation_file_key = lc(shift(@_));
467    # The username of the translation submitter
468    my $submitter_username = shift(@_);
469    # Whether to submit a target chunk even if it hasn't changed
470    my $force_submission_flag = shift(@_);
471
472    # Check that the necessary arguments were supplied
473    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
474    &log_message("Fatal error (but cannot be thrown): Missing command argument.");
475    die "\n";
476    }
477   
478    my %source_file_key_to_text_mapping = ();
479    my %source_file_key_to_gti_comment_mapping = ();
480    my %target_file_key_to_text_mapping = ();
481    my %target_file_key_to_gti_comment_mapping = ();
482
483    my (@source_file_lines, @target_file_lines) = ();
484    my ($source_file, $target_file, $translation_file_type);
485
486   
487    if ($translation_file_key ne "gs3interface") {
488    # Get (and check) the translation configuration
489    ($source_file, $target_file, $translation_file_type)
490        = &get_translation_configuration($target_language_code, $translation_file_key);
491
492    # Parse the source language and target language files
493    @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
494    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
495    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
496    %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
497
498    @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
499    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
500    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
501    %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
502    }
503    else {
504    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
505                 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
506    }
507    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
508    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
509
510    # Submission date
511    my $day = (localtime)[3];
512    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
513    my $year = (localtime)[5] + 1900;
514    my $submission_date = "$day-$month-$year";
515
516    open(SUBMISSION, "-");
517    my @submission_lines = <SUBMISSION>;
518    close(SUBMISSION);
519
520    # Remove any nasty carriage returns
521    # &log_message("Submission:");
522    foreach my $submission_line (@submission_lines) {
523    $submission_line =~ s/\r$//;
524    #&log_message("  $submission_line");
525    }
526
527    my %source_file_key_to_submission_mapping = ();
528    my %target_file_key_to_submission_mapping = ();
529    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
530    # Read source file part of submission
531    if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
532        my $chunk_key = $1;
533
534        # Read the source file text
535        my $source_file_chunk_text = "";
536        $i++;
537        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
538        $source_file_chunk_text .= $submission_lines[$i];
539        $i++;
540        }
541        $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
542        $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
543
544        #&log_message("Source file key: $chunk_key");
545        #&log_message("Source file text: $source_file_chunk_text");
546        $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
547    }
548
549    # Read target file part of submission
550    if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
551        my $chunk_key = $1;
552
553        # Read the target file text
554        my $target_file_chunk_text = "";
555        $i++;
556        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
557        $target_file_chunk_text .= $submission_lines[$i];
558        $i++;
559        }
560        $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
561        $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
562
563        #&log_message("Target file key: $chunk_key");
564        #&log_message("Target file text: $target_file_chunk_text");
565        $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
566    }
567    }
568
569    # -----------------------------------------
570    #   Validate the translation submissions
571    # -----------------------------------------
572
573    # Check that the translations are valid
574    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
575    # Make sure the submitted chunk still exists in the source file
576    if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
577        &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
578        delete $source_file_key_to_submission_mapping{$chunk_key};
579        delete $target_file_key_to_submission_mapping{$chunk_key};
580        next;
581    }
582
583    # Make sure the submitted source chunk matches the source file chunk
584    if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
585        &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
586        &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
587        &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
588        delete $source_file_key_to_submission_mapping{$chunk_key};
589        delete $target_file_key_to_submission_mapping{$chunk_key};
590        next;
591    }
592    }
593
594    # Apply the submitted translations
595    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
596    # Only apply the submission if it is a change, unless -force_submission has been specified
597    if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
598        $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
599        $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
600    }
601    }
602   
603    if ($translation_file_key ne "gs3interface") {
604    eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
605    } else {
606    eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
607    }
608}
609
610
611sub create_glihelp_zip_file
612{
613    my $target_language_code = shift(@_);
614    my $translation_file_key = "glihelp";
615   
616    &log_message("Creating GLI Help zip file for $target_language_code");
617
618    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
619   
620    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
621    if ( ! -e $classpath) {
622    &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
623    }   
624   
625    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
626    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
627   
628    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
629    if ( ! -e $gen_many_html_xsl_filepath) {
630    &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
631    }
632
633    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
634    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
635   
636    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
637    $target_file_directory = $target_file_directory."/";
638
639    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
640    my $cmd = "/opt/jdk1.6.0/bin/java -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | perl -S $split_script_filepath $target_file_directory";
641    my $response = `$cmd`;
642    &log_message($cmd);
643    &log_message($response);   
644    &log_message("Created HTML operational files");
645   
646    $cmd = "/opt/jdk1.6.0/bin/java -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
647    $response = `$cmd`;
648    &log_message($cmd);
649    &log_message($response);   
650    &log_message("Created xml index file");
651
652    my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
653    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
654    $response = `$cmd`;
655    &log_message($cmd);
656    &log_message($response);   
657    &log_message("Created the zip file");
658}
659
660
661sub get_translation_configuration
662{
663    # Get the code of the target language
664    my $target_language_code = shift(@_);
665    # Get the key of the file to translate
666    my $translation_file_key = shift(@_);
667
668    # Read the translation data from the gti.cfg file
669    my ($source_file, $target_file, $translation_file_type) =
670    &get_translation_data_for($target_language_code, $translation_file_key);
671
672    # Check that the file to translate is defined in the gti.cfg file
673    if (!$source_file || !$target_file || !$translation_file_type) {
674    &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
675    }
676
677    # Check that the source file exists
678    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
679    if (!-e $source_file_path) {
680    &throw_fatal_error("Source file $source_file_path does not exist.");
681    }
682
683    # Check that the source file is up to date
684    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
685    # unless ($translation_file_is_not_in_cvs) {
686    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
687        my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
688    if ($source_file_cvs_status =~ /^C /) {
689        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
690    }
691    if ($source_file_cvs_status =~ /^M /) {
692        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
693    }
694    # }
695
696    return ($source_file, $target_file, $translation_file_type);
697}
698
699
700sub get_translation_data_for
701{
702    my ($target_language_code, $translation_file_key) = @_;
703
704    foreach my $translation_file (@$gti_translation_files) {
705    # If this isn't the correct translation file, move onto the next one
706    next if ($translation_file_key ne $translation_file->{'key'});
707
708    # Resolve the target language file
709    my $target_language_file = $translation_file->{'target_file'};
710    if ($target_language_file =~ /(\{.+\;.+\})/) {
711        my $unresolved_target_language_file_part = $1;
712
713        # Check for a special case for the target language code
714        if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
715        my $resolved_target_language_file_part = $2;
716        $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
717        }
718        # Otherwise use the last part as the default value
719        else {
720        my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
721        $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
722        }
723    }
724
725    # Resolve instances of {iso_639_1_target_language_name}
726    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
727    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
728    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
729
730    # Resolve instances of {target_language_code}
731    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
732
733    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
734    }
735
736    return ();
737}
738
739
740sub read_file_lines
741{
742    my ($file_path) = @_;
743
744    if (!open(FILE_IN, "<$file_path")) {
745    &log_message("Note: Could not open file $file_path.");
746    return ();
747    }
748    my @file_lines = <FILE_IN>;
749    close(FILE_IN);
750
751    return @file_lines;
752}
753
754
755sub build_key_to_line_mapping
756{
757    my ($file_lines, $translation_file_type) = @_;
758    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
759}
760
761
762sub build_key_to_text_mapping
763{
764    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
765
766    my %key_to_text_mapping = ();
767    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
768    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
769    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
770
771    my $chunk_text = @$file_lines[$chunk_starting_line];
772    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
773        $chunk_text .= @$file_lines[$l];
774    }
775
776    # Map from chunk key to text
777    eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
778    }
779
780    return %key_to_text_mapping;
781}
782
783
784sub build_key_to_last_update_date_mapping
785{
786    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
787
788    # If the files aren't in CVS then we can't tell anything about what needs updating
789    # return () if ($translation_file_is_not_in_cvs);
790
791    # Build a mapping from key to CVS date
792    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
793    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
794
795    # Build a mapping from key to comment date
796    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
797
798    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
799    my %key_to_last_update_date_mapping = ();
800    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
801    # Use the CVS date as a starting point
802    my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
803    $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
804
805    # If a comment date exists and it is after the CVS date, use that instead
806        # need to convert the comment date format to SVN format
807    my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
808    if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
809        my $chunk_comment_date = $1;           
810        # if the changes were made on 2009 Jan 27, ie. tidying up the GTI commands following each fragment,
811        # then should use the GTI comment date instead
812        # if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
813        if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date)
814         || (&is_date_after($chunk_cvs_date,$chunk_comment_date) && ($chunk_cvs_date eq "2009-01-27" || $chunk_cvs_date eq "2009-01-28")))) {
815        $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
816       
817        # &log_message("cvs date: $chunk_cvs_date, comment date: $chunk_comment_date");
818        # die;
819        }
820    }
821    }
822
823    return %key_to_last_update_date_mapping;
824}
825
826
827sub build_key_to_cvs_date_mapping
828{
829    my ($filename, $translation_file_type) = @_;
830
831    # Use CVS to annotate each line of the file with the date it was last edited
832    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
833    # my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;   
834    # my $cvs_annotated_file = `cd $gsdl_root_directory; export PATH=.:/research/lh92/programs/subversion/bin; svn annotate -v --force $filename`;
835    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v $filename`;
836   
837    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
838
839    my @cvs_annotated_file_lines_date = ();
840    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
841    # Extract the date from the CVS annotation at the front
842        # cvs format : 07-Jun-02
843        # svn format : 2007-07-16
844    # $cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
845        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
846       
847        push(@cvs_annotated_file_lines_date, $1);
848       
849        # trim extra date information in svn annotation format
850        # 15:42:49 +1200 (Wed, 21 Jun 2006)
851        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
852    }   
853   
854    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
855    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
856       
857    my %key_to_cvs_date_mapping = ();
858    foreach my $chunk_key (keys(%key_to_line_mapping)) {
859    my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
860    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
861               
862    # Find the date this chunk was last edited, from the CVS annotation
863    my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
864    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
865        if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
866        # This part of the chunk has been updated more recently
867        $chunk_date = $cvs_annotated_file_lines_date[$l];
868       
869        }
870    }
871
872    # Map from chunk key to CVS date
873    $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
874    }
875
876    return %key_to_cvs_date_mapping;
877}
878
879
880sub build_key_to_gti_comment_mapping
881{
882    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
883
884    my %key_to_gti_comment_mapping = ();
885    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
886    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
887    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
888
889    my $chunk_text = @$file_lines[$chunk_starting_line];
890    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
891        $chunk_text .= @$file_lines[$l];
892    }
893
894    # Map from chunk key to GTI comment
895    my $chunk_gti_comment;
896    eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
897    $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
898    }
899
900    return %key_to_gti_comment_mapping;
901}
902
903
904sub determine_chunks_requiring_translation
905{
906    my $source_file_key_to_text_mapping = shift(@_);
907    my $target_file_key_to_text_mapping = shift(@_);
908
909    # Chunks needing translation are those in the source file with no translation in the target file
910    my @target_file_keys_requiring_translation = ();
911    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
912    if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
913        # &log_message("Chunk with key $chunk_key needs translating.");
914        push(@target_file_keys_requiring_translation, $chunk_key);
915    }
916    }
917
918    return @target_file_keys_requiring_translation;
919}
920
921
922sub determine_chunks_requiring_updating
923{
924    my $source_file_key_to_last_update_date_mapping = shift(@_);
925    my $target_file_key_to_last_update_date_mapping = shift(@_);
926
927    # Chunks needing updating are those in the target file that have been more recently edited in the source file
928    my @target_file_keys_requiring_updating = ();
929    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
930    my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
931    my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
932       
933        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
934                   
935        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
936        # &log_message("Chunk with key $chunk_key needs updating.");
937        push(@target_file_keys_requiring_updating, $chunk_key);
938    }
939    }
940
941    return @target_file_keys_requiring_updating;
942}
943
944
945sub is_chunk_automatically_translated
946{
947    my ($chunk_key, $translation_file_type) = @_;
948    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
949}
950
951
952sub make_text_xml_safe
953{
954    my $text = shift(@_);
955    $text =~ s/\&/\&amp\;/g;
956    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
957    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
958    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
959    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
960    $text =~ s/</\&lt\;/g;
961    $text =~ s/>/\&gt\;/g;
962    return $text;
963}
964
965
966sub unmake_text_xml_safe
967{
968    my $text = shift(@_);
969    $text =~ s/\&lt\;/</g;
970    $text =~ s/\&gt\;/>/g;
971    $text =~ s/\&amp\;/\&/g;
972    return $text;
973}
974
975
976# Returns 1 if $date1 is after $date2, 0 otherwise
977sub is_date_after_cvs
978{
979    my ($date1, $date2) = @_;
980    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
981          "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
982
983    if(!defined $date1) {
984        return 1;
985    }
986
987    my @date1parts = split(/-/, $date1);
988    my @date2parts = split(/-/, $date2);
989
990    # Compare year - nasty because we have rolled over into a new century
991    my $year1 = $date1parts[2];
992    if ($year1 < 80) {
993        $year1 += 2000;
994    }
995    my $year2 = $date2parts[2];
996    if ($year2 < 80) {
997        $year2 += 2000;
998    }
999
1000    # Compare year
1001    if ($year1 > $year2) {
1002    return 1;
1003    }
1004    elsif ($year1 == $year2) {
1005    # Year is the same, so compare month
1006    if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1007        return 1;
1008    }
1009    elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1010        # Month is the same, so compare day
1011        if ($date1parts[0] > $date2parts[0]) {
1012        return 1;
1013        }
1014    }
1015    }
1016
1017    return 0;
1018}
1019
1020sub is_date_after
1021{
1022    my ($date1, $date2) = @_;
1023   
1024    if(!defined $date1) {
1025      return 1;
1026    }
1027    if(!defined $date2) {
1028      return 0;
1029    }
1030   
1031    # 16-Aug-2006
1032    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1033       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1034          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1035       $date1=$3 . "-" . $months{$2} . "-" . $1;
1036       # print "** converted date1: $date1\n";
1037    }
1038    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1039       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1040          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1041       $date2=$3 . "-" . $months{$2} . "-" . $1;
1042       # print "** converted date2: $date2\n";
1043    }
1044   
1045   
1046    # 2006-08-16
1047    my @date1parts = split(/-/, $date1);
1048    my @date2parts = split(/-/, $date2);
1049   
1050    # Compare year
1051    if ($date1parts[0] > $date2parts[0]) {
1052    return 1;
1053    }
1054    elsif ($date1parts[0] == $date2parts[0]) {
1055    # Year is the same, so compare month
1056    if ($date1parts[1] > $date2parts[1]) {
1057        return 1;
1058    }
1059    elsif ($date1parts[1] == $date2parts[1]) {
1060        # Month is the same, so compare day
1061        if ($date1parts[2] > $date2parts[2]) {
1062        return 1;
1063        }
1064    }
1065    }   
1066   
1067    return 0;
1068}
1069
1070
1071sub create_xml_response_for_chunks_requiring_work
1072{
1073    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1074
1075    # Form an XML response to the command
1076    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1077    $xml_response .= "<GTIResponse>\n";
1078    $xml_response .= "  <TranslationFile"
1079    . " key=\"" . $translation_file_key . "\""
1080    . " target_file_path=\"" . $target_file . "\""
1081    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1082    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1083    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1084
1085    # Do chunks requiring translation first
1086    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1087    $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1088    }
1089    else {
1090    $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1091    }
1092
1093    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1094    foreach my $chunk_key (@sorted_chunk_keys) {
1095    last if ($num_chunks_to_return == 0);
1096
1097    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1098    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1099   
1100    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1101    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1102    $xml_response .= "      <TargetFileText></TargetFileText>\n";
1103    $xml_response .= "    </Chunk>\n";
1104
1105    $num_chunks_to_return--;
1106    }
1107
1108    $xml_response .= "  </ChunksRequiringTranslation>\n";
1109
1110    # Then do chunks requiring updating
1111    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1112    $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1113    }
1114    else {
1115    $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1116    }
1117       
1118    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1119    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1120    foreach my $chunk_key (@sorted_chunk_keys) {
1121    last if ($num_chunks_to_return == 0);
1122
1123    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1124    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1125    my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1126    my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1127
1128    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1129    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1130    $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1131    $xml_response .= "    </Chunk>\n";
1132
1133    $num_chunks_to_return--;
1134    }
1135
1136    $xml_response .= "  </ChunksRequiringUpdating>\n";
1137
1138    $xml_response .= "</GTIResponse>\n";
1139
1140    return $xml_response;
1141}
1142
1143
1144sub create_xml_response_for_all_chunks
1145{
1146    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1147
1148    # Form an XML response to the command
1149    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1150    $xml_response .= "<GTIResponse>\n";
1151    $xml_response .= "  <TranslationFile"
1152    . " key=\"" . $translation_file_key . "\""
1153    . " target_file_path=\"" . $target_file . "\"\/>\n";
1154   
1155    # Do all the chunks
1156    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1157
1158    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1159    foreach my $chunk_key (@sorted_chunk_keys) {
1160    my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1161    my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1162
1163    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1164    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1165    if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1166        my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1167        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1168        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1169    }
1170    else {
1171        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1172    }
1173
1174    $xml_response .= "    </Chunk>\n";
1175    }
1176    $xml_response .= "  </Chunks>\n";
1177   
1178    $xml_response .= "</GTIResponse>\n";
1179    return $xml_response;
1180}
1181
1182
1183
1184# ==========================================================================================
1185#   MACROFILE FUNCTIONS
1186
1187sub build_key_to_line_mapping_for_macrofile
1188{
1189    my (@file_lines) = @_;
1190
1191    my $macro_package;
1192    my %chunk_key_to_line_mapping = ();
1193    # Process the contents of the file, line by line
1194    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1195    my $line = $file_lines[$i];
1196    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1197
1198    # Check if a new package is being defined
1199    if ($line =~ m/^package\s+(.+)/) {
1200        $macro_package = $1;
1201    }
1202
1203    # Line contains a macro name
1204    elsif ($line =~ m/^(_\w+_)/) {
1205        my $macro_key = $1;
1206        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1207
1208        # While there is still text of the macro to go...
1209        my $startline = $i;
1210        while ($line !~ /\}$/) {
1211        $i++;
1212        if ($i == scalar(@file_lines)) {
1213            &throw_fatal_error("Could not find end of macro $macro_key.");
1214        }
1215        $line = $file_lines[$i];
1216        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1217        }
1218
1219        # The chunk key consists of the package name and the macro key
1220        my $chunk_key = $macro_package . "." . $macro_key;
1221        # Map from chunk key to line
1222        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1223    }
1224
1225    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1226    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1227        # The chunk key consists of package name and macro key
1228        my $chunk_key = $macro_package . "." . $1;
1229        # Map from chunk key to line
1230        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1231    }
1232    }
1233
1234    return %chunk_key_to_line_mapping;
1235}
1236
1237
1238sub import_chunk_from_macrofile
1239{
1240    my ($chunk_text) = @_;
1241
1242    # Is this an icon macro??
1243    if ($chunk_text =~ /^\#\# (.*)/) {
1244    # Extract image macro text
1245    $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1246    $chunk_text = $1;
1247
1248    # Remove enclosing quotes
1249    $chunk_text =~ s/^\"//;
1250    $chunk_text =~ s/\"$//;
1251    }
1252
1253    # No, so it must be a text macro
1254    else {
1255    # Remove macro key
1256    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1257
1258    # Remove language specifier
1259    $chunk_text =~ s/^\[l=.*\](\s*)//;
1260
1261    # Remove braces enclosing text
1262    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1263    }
1264
1265    return $chunk_text;
1266}
1267
1268
1269sub get_macrofile_chunk_gti_comment
1270{
1271    my ($chunk_text) = @_;
1272
1273    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1274    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1275    return $1;
1276    }
1277
1278    return undef;
1279}
1280
1281
1282sub is_macrofile_chunk_automatically_translated
1283{
1284    my ($chunk_key) = @_;
1285
1286    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1287    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1288    return 1;
1289    }
1290
1291    return 0;
1292}
1293
1294
1295# Use the source file to generate a target file that is formatted the same
1296sub write_translated_macrofile
1297{
1298    my $source_file = shift(@_);  # Not used
1299    my @source_file_lines = @{shift(@_)};
1300    my $source_file_key_to_text_mapping = shift(@_);
1301    my $target_file = shift(@_);
1302    my @target_file_lines = @{shift(@_)};
1303    my $target_file_key_to_text_mapping = shift(@_);
1304    my $target_file_key_to_gti_comment_mapping = shift(@_);
1305    my $target_language_code = shift(@_);
1306
1307    # Build a mapping from source file line to chunk key
1308    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1309    my %source_file_line_to_key_mapping = ();
1310    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1311    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1312    }
1313    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1314    my $source_file_line_number = 0;
1315
1316    # Build a mapping from target file line to chunk key
1317    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1318    my %target_file_line_to_key_mapping = ();
1319    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1320    $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1321    }
1322    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1323
1324    # Write the new target file
1325    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1326    if (!open(TARGET_FILE, ">$target_file_path")) {
1327    &throw_fatal_error("Could not write target file $target_file_path.");
1328    }
1329
1330    # Use the header from the target file, to keep language and author information
1331    if (scalar(@target_file_line_keys) > 0) {
1332    my $target_file_line_number = 0;
1333    my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1334    while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1335        my $target_file_line = $target_file_lines[$target_file_line_number];
1336        last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1337        print TARGET_FILE $target_file_line;
1338        $target_file_line_number++;
1339    }
1340
1341    $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1342    }
1343
1344    # Model the new target file on the source file, with the target file translations
1345    foreach my $line_key (@source_file_line_keys) {
1346    # Fill in the gaps before this chunk starts
1347    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1348    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1349    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1350        print TARGET_FILE $source_file_lines[$source_file_line_number];
1351        $source_file_line_number++;
1352    }
1353    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1354
1355    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1356    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1357    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1358
1359    my $macrofile_key = $chunk_key;
1360    $macrofile_key =~ s/^(.+?)\.//;
1361
1362    # If no translation exists for this chunk, show this, and move on
1363    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1364        print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1365        next;
1366    }
1367
1368    # Grab the source chunk text
1369    my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1370    for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1371        $source_file_chunk .= $source_file_lines[$l];
1372    }
1373
1374    # Is this an icon macro??
1375    if ($source_file_chunk =~ /^\#\# (.*)/) {
1376        # Escape any newline and question mark characters so the source text is replaced correctly
1377        $source_file_chunk_text =~ s/\\/\\\\/g;
1378        $source_file_chunk_text =~ s/\?/\\\?/g;
1379
1380        # Build the new target chunk from the source chunk
1381        my $target_file_chunk = $source_file_chunk;
1382        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1383        $target_file_chunk =~ s/(\s)*$//;
1384        print TARGET_FILE "$target_file_chunk";
1385    }
1386
1387    # No, it is just a normal text macro
1388    else {
1389        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1390    }
1391
1392    # Add the "updated" comment, if one exists
1393    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1394        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1395    }
1396    print TARGET_FILE "\n";
1397    }
1398
1399    close(TARGET_FILE);
1400}
1401
1402
1403sub sort_by_line
1404{
1405    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1406}
1407
1408
1409# ==========================================================================================
1410#   RESOURCE BUNDLE FUNCTIONS
1411
1412sub build_key_to_line_mapping_for_resource_bundle
1413{
1414    my (@file_lines) = @_;
1415
1416    my %chunk_key_to_line_mapping = ();
1417    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1418    my $line = $file_lines[$i];
1419    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1420
1421    # Line contains a dictionary string
1422    if ($line =~ /^(\S+?)[:|=](.*)$/) {
1423        my $chunk_key = $1;
1424
1425        # Map from chunk key to line
1426        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1427    }
1428    }
1429
1430    return %chunk_key_to_line_mapping;
1431}
1432
1433
1434sub import_chunk_from_resource_bundle
1435{
1436    my ($chunk_text) = @_;
1437
1438    # Simple: just remove string key
1439    $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1440    $chunk_text =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1441    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1442
1443    return $chunk_text;
1444}
1445
1446
1447sub get_resource_bundle_chunk_gti_comment
1448{
1449    my ($chunk_text) = @_;
1450
1451    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1452    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1453    return $1;
1454    }
1455
1456    return undef;
1457}
1458
1459
1460sub is_resource_bundle_chunk_automatically_translated
1461{
1462    # No resource bundle chunks are automatically translated
1463    return 0;
1464}
1465
1466
1467sub write_translated_resource_bundle
1468{
1469    my $source_file = shift(@_);  # Not used
1470    my @source_file_lines = @{shift(@_)};
1471    my $source_file_key_to_text_mapping = shift(@_);
1472    my $target_file = shift(@_);
1473    my @target_file_lines = @{shift(@_)};  # Not used
1474    my $target_file_key_to_text_mapping = shift(@_);
1475    my $target_file_key_to_gti_comment_mapping = shift(@_);
1476    my $target_language_code = shift(@_);  # Not used
1477
1478    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1479    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1480    my %source_file_line_to_key_mapping = ();
1481    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1482    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1483    }
1484
1485    # Write the new target file
1486    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1487    if (!open(TARGET_FILE, ">$target_file_path")) {
1488    &throw_fatal_error("Could not write target file $target_file_path.");
1489    }
1490
1491    # Model the new target file on the source file, with the target file translations
1492    my $source_file_line_number = 0;
1493    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1494    # Fill in the gaps before this chunk starts
1495    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1496    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1497    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1498        print TARGET_FILE $source_file_lines[$source_file_line_number];
1499        $source_file_line_number++;
1500    }
1501    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1502
1503    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1504    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1505    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1506
1507    # If no translation exists for this chunk, show this, and move on
1508    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1509        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1510        next;
1511    }
1512
1513    print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1514    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1515        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1516    }
1517    print TARGET_FILE "\n";
1518    }
1519
1520    close(TARGET_FILE);
1521}
1522
1523
1524# ==========================================================================================
1525#   GREENSTONE XML FUNCTIONS
1526
1527sub build_key_to_line_mapping_for_greenstone_xml
1528{
1529    my (@file_lines) = @_;
1530
1531    my %chunk_key_to_line_mapping = ();
1532    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1533    my $line = $file_lines[$i];
1534    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1535
1536    # Line contains a string to translate
1537    if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1538        my $chunk_key = $1;
1539        $line =~ s/\s*$//;  # Remove any nasty whitespace
1540        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1541
1542        # While there is still text of the string to go...
1543        my $startline = $i;
1544        while ($line !~ /<\/Text>$/) {
1545        $i++;
1546        if ($i == scalar(@file_lines)) {
1547            &throw_fatal_error("Could not find end of string $chunk_key.");
1548        }
1549        $line = $file_lines[$i];
1550        $line =~ s/\s*$//;  # Remove any nasty whitespace
1551        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1552        }
1553
1554        # Map from chunk key to line
1555        if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1556        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1557        }
1558        else {
1559        &throw_fatal_error("Duplicate key $chunk_key.");
1560        }
1561    }
1562    }
1563
1564    return %chunk_key_to_line_mapping;
1565}
1566
1567
1568sub import_chunk_from_greenstone_xml
1569{
1570    my ($chunk_text) = @_;
1571
1572    # Simple: just remove the Text tags
1573    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1574    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1575    $chunk_text =~ s/<\/Text>$//;
1576
1577    return $chunk_text;
1578}
1579
1580
1581sub get_greenstone_xml_chunk_gti_comment
1582{
1583    my ($chunk_text) = @_;
1584
1585    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1586    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1587    return $1;
1588    }
1589
1590    return undef;
1591}
1592
1593
1594sub is_greenstone_xml_chunk_automatically_translated
1595{
1596    # No greenstone XML chunks are automatically translated
1597    return 0;
1598}
1599
1600
1601sub write_translated_greenstone_xml
1602{
1603    my $source_file = shift(@_);  # Not used
1604    my @source_file_lines = @{shift(@_)};
1605    my $source_file_key_to_text_mapping = shift(@_);
1606    my $target_file = shift(@_);
1607    my @target_file_lines = @{shift(@_)};  # Not used
1608    my $target_file_key_to_text_mapping = shift(@_);
1609    my $target_file_key_to_gti_comment_mapping = shift(@_);
1610    my $target_language_code = shift(@_);  # Not used
1611
1612    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1613    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1614    my %source_file_line_to_key_mapping = ();
1615    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1616    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1617    }
1618
1619    # Write the new target file
1620    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1621    if (!open(TARGET_FILE, ">$target_file_path")) {
1622    &throw_fatal_error("Could not write target file $target_file_path.");
1623    }
1624
1625    # Model the new target file on the source file, with the target file translations
1626    my $source_file_line_number = 0;
1627    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1628    # Fill in the gaps before this chunk starts
1629    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1630    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1631    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1632        print TARGET_FILE $source_file_lines[$source_file_line_number];
1633        $source_file_line_number++;
1634    }
1635    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1636
1637    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1638    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1639    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1640    $target_file_chunk_text =~ s/(\n)*$//g;
1641
1642    # If no translation exists for this chunk, show this, and move on
1643    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1644        print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1645        next;
1646    }
1647
1648    print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1649    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1650        my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1651        $chunk_gti_comment =~ s/^Updated //;
1652        print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1653    }
1654    print TARGET_FILE "\n";
1655    }
1656
1657    # Fill in the end of the file
1658    while ($source_file_line_number < scalar(@source_file_lines)) {
1659    print TARGET_FILE $source_file_lines[$source_file_line_number];
1660    $source_file_line_number++;
1661    }
1662
1663    close(TARGET_FILE);
1664}
1665
1666
1667# ==========================================================================================
1668#   GREENSTONE3 FUNCTIONS
1669
1670sub get_all_chunks_for_gs3
1671{
1672    # The code of the target language (ensure it is lowercase)
1673    my $target_language_code = lc(shift(@_));
1674    my $translation_file_key = lc(shift(@_));
1675
1676    # Check that the necessary arguments were supplied
1677    if (!$target_language_code) {
1678    &throw_fatal_error("Missing command argument.");
1679    }
1680
1681    # Get (and check) the translation configuration
1682    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1683   
1684    my %source_files_key_to_text_mapping = ();
1685    my %target_files_key_to_text_mapping = ();
1686    my %source_files_key_to_last_update_date_mapping = ();
1687    my %target_files_key_to_last_update_date_mapping = ();
1688
1689    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1690
1691    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1692    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1693
1694    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1695    return $xml_response;
1696}
1697
1698
1699sub get_first_n_chunks_requiring_work_for_gs3
1700{
1701    # The code of the target language (ensure it is lowercase)
1702    my $target_language_code = lc(shift(@_));
1703    # The key of the file to translate (ensure it is lowercase)
1704    my $translation_file_key = lc(shift(@_));
1705    # The number of chunks to return (defaults to one if not specified)
1706    my $num_chunks_to_return = shift(@_) || "1";
1707   
1708    # Check that the necessary arguments were supplied
1709    if (!$target_language_code || !$translation_file_key) {
1710    &throw_fatal_error("Missing command argument.");
1711    }
1712   
1713    my %source_files_key_to_text_mapping = ();
1714    my %target_files_key_to_text_mapping = ();
1715    my %source_files_key_to_last_update_date_mapping = ();
1716    my %target_files_key_to_last_update_date_mapping = ();
1717
1718    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1719                 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1720   
1721    # Determine the target file chunks requiring translation
1722    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1723    # Determine the target file chunks requiring updating
1724    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1725    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1726    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1727
1728    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1729                                      \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1730                                      $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1731                                      \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1732   
1733    return $xml_response;
1734}
1735
1736
1737
1738sub build_gs3_configuration
1739{
1740    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1741    $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1742   
1743    my $source_file_directory = "greenstone3";
1744    my $translation_file_type = "resource_bundle";
1745
1746    foreach my $interface_file_key (@gs3_interface_files) {
1747
1748    &log_message("Greenstone 3 interface file: " . $interface_file_key);
1749
1750    # Parse the source language and target language files
1751    my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1752    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1753    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1754    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1755    my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1756   
1757    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1758    my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1759    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1760    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1761    my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1762
1763
1764    # Filter out any automatically translated chunks
1765    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1766        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1767        delete $source_file_key_to_line_mapping{$chunk_key};
1768        delete $target_file_key_to_line_mapping{$chunk_key};
1769        }
1770    }
1771
1772    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1773    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1774
1775    foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1776        my $global_chunk_key = "$interface_file_key.$chunk_key";
1777        $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1778        $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1779
1780        if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1781        $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1782        $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1783        }
1784    }   
1785    }
1786}
1787
1788
1789sub write_translated_gs3interface
1790{
1791    my $source_file_key_to_text_mapping = shift(@_);
1792    my $target_file_key_to_text_mapping = shift(@_);
1793    my $target_file_key_to_gti_comment_mapping = shift(@_);
1794    my $target_language_code = shift(@_);
1795   
1796    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1797
1798    my %translated_interface_file_keys = ();
1799    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1800    $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1801    if (!defined $translated_interface_file_keys{$1}) {
1802        &log_message("Updated interface file: " . $1); 
1803        $translated_interface_file_keys{$1}="";
1804    }
1805    }
1806    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1807   
1808    my $source_file_directory = "greenstone3";   
1809   
1810    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1811   
1812    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1813    my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1814    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1815    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1816    my %source_file_line_to_key_mapping = ();
1817    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1818        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1819    }
1820   
1821    # Write the new target file
1822    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1823    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1824    if (!open(TARGET_FILE, ">$target_file_path")) {
1825        &throw_fatal_error("Could not write target file $target_file_path.");
1826    }
1827
1828    # Model the new target file on the source file, with the target file translations
1829    my $source_file_line_number = 0;
1830    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1831        # Fill in the gaps before this chunk starts
1832        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1833        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1834        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1835        print TARGET_FILE $source_file_lines[$source_file_line_number];
1836        $source_file_line_number++;
1837        }
1838        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1839
1840        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1841        my $global_chunk_key = "$interface_file_key.$chunk_key";
1842        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1843        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1844
1845        # If no translation exists for this chunk, show this, and move on
1846        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1847        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1848        next;
1849        }
1850
1851        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1852        if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1853        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1854        }
1855        print TARGET_FILE "\n";
1856    }
1857
1858    close(TARGET_FILE);
1859    }           
1860}
1861
1862&main(@ARGV);
Note: See TracBrowser for help on using the browser.