root/main/trunk/greenstone2/bin/script/gti.pl @ 24093

Revision 24093, 77.1 KB (checked in by sjm84, 8 years ago)

Fixing issues with perl finding the wrong perl by making sure it uses the one that is currently running

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33
34    if(!$ENV{'PERLPATH'}) {
35    my $full_perl_exec = $^X;
36    require File::Basename;
37    my $perl_path = File::Basename::dirname($full_perl_exec);
38    $ENV{'PERLPATH'} = $perl_path;
39    }
40}
41
42use iso639;
43use strict;
44use util;
45
46
47#my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
48#my $anonymous_svn_root = "http://http://svn.greenstone.org/gsdl/trunk/";
49my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
50my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
51my $source_language_code = "en";  # This is non-negiotable
52
53my $gti_translation_files =
54    [ # Greenstone macrofiles
55      { 'key' => "coredm",
56    'file_type' => "macrofile",
57    'source_file' => "macros/english.dm",
58    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
59      { 'key' => "auxdm",
60    'file_type' => "macrofile",
61    'source_file' => "macros/english2.dm",
62    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
63
64      # GLI dictionary
65      { 'key' => "glidict",
66    'file_type' => "resource_bundle",
67    'source_file' => "gli/classes/dictionary.properties",
68    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
69
70      # GLI help
71      { 'key' => "glihelp",
72    'file_type' => "greenstone_xml",
73    'source_file' => "gli/help/en/help.xml",
74    'target_file' => "gli/help/{target_language_code}/help.xml" },
75
76      # Greenstone Perl modules
77      { 'key' => "perlmodules",
78    'file_type' => "resource_bundle",
79    'source_file' => "perllib/strings.properties",
80    'target_file' => "perllib/strings_{target_language_code}.properties" },
81
82      # Greenstone tutorial exercises
83      # { 'key' => "tutorials",
84    # 'file_type' => "greenstone_xml",
85    # 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
86    # 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
87
88      # new Greenstone.org
89      { 'key' => "greenorg",
90        'file_type' => "resource_bundle",
91        'source_file' => "greenstoneorg/website/classes/Gsc.properties",
92        'target_file' => "greenstoneorg/website/classes/Gsc_{iso_639_1_target_language_name}.properties"
93    # 'file_type' => "macrofile",
94    # 'source_file' => "greenorg/macros/english.dm",
95    # 'target_file' => "greenorg/macros/{iso_639_1_target_language_name}.dm"
96      }
97
98      # { 'key' => "gs3interface",
99    #'file_type' => "resource_bundle",
100    #'source_file' => "greenstone3",
101    #'target_file' => "greenstone3"
102      #}
103    ];
104
105my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
106
107
108sub main
109{
110    # Get the command to process, and any arguments
111    my $gti_command = shift(@_);
112    my @gti_command_arguments = @_;
113    my $module = $_[1];
114
115    # Open the GTI log file for appending, or write to STDERR if that fails
116    if (!open(GTI_LOG, ">>$gti_log_file")) {
117    open(GTI_LOG, ">&STDERR");
118    }
119
120    # Log the command that launched this script
121    &log_message("Command: $0 @ARGV");
122
123    # Check that a command was supplied
124    if (!$gti_command) {
125    &throw_fatal_error("Missing command.");
126    }     
127
128    # Process the command
129    if ($gti_command =~ /^get-all-chunks$/i) {
130    # Check that GS3 interface is the target
131    if ($module eq "gs3interface") {       
132        print &get_all_chunks_for_gs3(@gti_command_arguments);
133    } else {
134        print &get_all_chunks(@gti_command_arguments);
135    }
136    }
137    if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
138    if ($module eq "gs3interface") {       
139        print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
140    } else {
141        print &get_first_n_chunks_requiring_work(@gti_command_arguments);
142    }
143    }
144    if ($gti_command =~ /^get-language-status$/i) {
145    print &get_language_status(@gti_command_arguments);       
146    }
147    if ($gti_command =~ /^search-chunks$/i) {
148    print &search_chunks(@gti_command_arguments);
149    }
150    if ($gti_command =~ /^submit-translations$/i) {
151    # This command cannot produce any output since it reads input
152    &submit_translations(@gti_command_arguments);
153    }
154    if ($gti_command =~ /^create-glihelp-zip-file$/i) {
155    # This command cannot produce any output since it reads input
156    &create_glihelp_zip_file(@gti_command_arguments);
157    }
158#    else {
159    # The command was not recognized
160    #&throw_fatal_error("Unknown command \"$gti_command\".");
161    #}
162}
163
164
165sub throw_fatal_error
166{
167    my $error_message = shift(@_);
168
169    # Write an XML error response
170    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
171    print "<GTIResponse>\n";
172    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
173    print "</GTIResponse>\n";
174
175    # Log the error message, then die
176    &log_message("Error: $error_message");
177    die "\n";
178}
179
180
181sub log_message
182{
183    my $log_message = shift(@_);
184    print GTI_LOG time() . " -- " . $log_message . "\n";
185}
186
187
188sub get_all_chunks
189{
190    # The code of the target language (ensure it is lowercase)
191    my $target_language_code = lc(shift(@_));
192    # The key of the file to translate (ensure it is lowercase)
193    my $translation_file_key = lc(shift(@_));
194
195    # Check that the necessary arguments were supplied
196    if (!$target_language_code || !$translation_file_key) {
197    &throw_fatal_error("Missing command argument.");
198    }
199
200    # Get (and check) the translation configuration
201    my ($source_file, $target_file, $translation_file_type)
202    = &get_translation_configuration($target_language_code, $translation_file_key);
203
204    # Parse the source language and target language files
205    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
206    my @source_file_lines = &read_file_lines($source_file_path);
207    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
208   
209    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
210    my @target_file_lines = &read_file_lines($target_file_path);
211    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
212
213    # Filter out any automatically translated chunks
214    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
215    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
216        delete $source_file_key_to_line_mapping{$chunk_key};
217        delete $target_file_key_to_line_mapping{$chunk_key};
218    }
219    }
220
221    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
222    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
223    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
224    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
225
226    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
227    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
228
229    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
230   
231    return $xml_response;
232}
233
234
235sub get_first_n_chunks_requiring_work
236{
237    # The code of the target language (ensure it is lowercase)
238    my $target_language_code = lc(shift(@_));
239    # The key of the file to translate (ensure it is lowercase)
240    my $translation_file_key = lc(shift(@_));
241    # The number of chunks to return (defaults to one if not specified)
242    my $num_chunks_to_return = shift(@_) || "1";
243
244    # Check that the necessary arguments were supplied
245    if (!$target_language_code || !$translation_file_key) {
246    &throw_fatal_error("Missing command argument.");
247    }
248
249    # Get (and check) the translation configuration
250    my ($source_file, $target_file, $translation_file_type)
251    = &get_translation_configuration($target_language_code, $translation_file_key);
252
253    # Parse the source language and target language files
254    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
255    my @source_file_lines = &read_file_lines($source_file_path);
256    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
257   
258    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
259    my @target_file_lines = &read_file_lines($target_file_path);
260    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
261
262    # Filter out any automatically translated chunks
263    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
264    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
265        delete $source_file_key_to_line_mapping{$chunk_key};
266        delete $target_file_key_to_line_mapping{$chunk_key};
267    }
268    }
269
270    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
271    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
272    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
273    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
274
275    # Determine the target file chunks requiring translation
276    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
277    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
278
279    # Determine the target file chunks requiring updating
280    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
281    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
282    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
283    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
284
285    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
286   
287    return $xml_response;
288}
289
290
291sub get_language_status
292{
293    # The code of the target language (ensure it is lowercase)
294    my $target_language_code = lc(shift(@_));
295
296    # Check that the necessary arguments were supplied
297    if (!$target_language_code) {
298    &throw_fatal_error("Missing command argument.");
299    }
300
301    # Form an XML response to the command
302    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
303    $xml_response .= "<GTIResponse>\n";
304    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
305
306    foreach my $translation_file (@$gti_translation_files) {   
307    my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
308    my $target_file_name = "";
309   
310    if ($translation_file->{'key'} eq "gs3interface") {
311        my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
312        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
313       
314        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
315        my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
316
317        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
318        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
319        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
320        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
321    }
322    else {
323        # Get (and check) the translation configuration
324        my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
325        $target_file_name = $target_file;
326
327        # Parse the source language and target language files
328        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
329        my @source_file_lines = &read_file_lines($source_file_path);
330        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
331
332        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
333        my @target_file_lines = &read_file_lines($target_file_path);
334        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
335
336        # Filter out any automatically translated chunks
337        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
338        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
339            delete $source_file_key_to_line_mapping{$chunk_key};
340            delete $target_file_key_to_line_mapping{$chunk_key};
341        }
342        }
343
344        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
345        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
346
347        # Determine the target file chunks requiring translation
348        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
349
350        # Determine the target file chunks requiring updating
351        my @target_file_keys_requiring_updating = ();
352        if (-e $target_file_path) {
353        my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
354        my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
355        @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
356        }
357
358        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
359        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
360        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
361        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
362    }
363   
364    &log_message("Status of " . $translation_file->{'key'});
365    &log_message("Number of source chunks: " . $num_source_chunks);
366    &log_message("Number of target chunks: " . $num_target_chunks);
367    &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
368    &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
369
370    $xml_response .= "    <TranslationFile"
371        . " key=\"" . $translation_file->{'key'} . "\""
372        . " target_file_path=\"" . $target_file_name . "\""
373        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
374        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
375        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
376    }
377
378    $xml_response .= "  </LanguageStatus>\n";
379
380    $xml_response .= "</GTIResponse>\n";
381    return $xml_response;
382}
383
384
385sub search_chunks
386{
387    # The code of the target language (ensure it is lowercase)
388    my $target_language_code = lc(shift(@_));
389    # The key of the file to translate (ensure it is lowercase)
390    my $translation_file_key = lc(shift(@_));
391    # The query string
392    my $query_string = join(' ', @_);
393
394    # Check that the necessary arguments were supplied
395    if (!$target_language_code || !$translation_file_key || !$query_string) {
396    &throw_fatal_error("Missing command argument.");
397    }
398
399    my ($source_file, $target_file, $translation_file_type) = ();
400    my %source_file_key_to_text_mapping = ();
401    my %target_file_key_to_text_mapping = ();
402   
403   
404    if ($translation_file_key ne "gs3interface") {
405    # Get (and check) the translation configuration
406    ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
407
408    # Parse the source language and target language files
409    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
410    my @source_file_lines = &read_file_lines($source_file_path);
411    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
412
413    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
414    my @target_file_lines = &read_file_lines($target_file_path);
415    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
416
417    # Filter out any automatically translated chunks
418    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
419        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
420        delete $source_file_key_to_line_mapping{$chunk_key};
421        delete $target_file_key_to_line_mapping{$chunk_key};
422        }
423    }
424
425    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
426    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
427    }
428    else {
429    # Not needed in this case
430    my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
431    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
432                 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
433    }
434
435    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
436    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
437
438    # Determine the target file chunks matching the query
439    my @target_file_keys_matching_query = ();
440    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
441    my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
442    if ($target_file_text =~ /$query_string/i) {
443        # &log_message("Chunk with key $chunk_key matches query.");
444        push(@target_file_keys_matching_query, $chunk_key);
445    }
446    }
447
448    # Form an XML response to the command
449    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
450    $xml_response .= "<GTIResponse>\n";
451
452    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
453    foreach my $chunk_key (@target_file_keys_matching_query) {
454    my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
455
456    $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
457    $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
458    $xml_response .= "    </Chunk>\n";
459    }
460    $xml_response .= "  </ChunksMatchingQuery>\n";
461
462    $xml_response .= "</GTIResponse>\n";
463    return $xml_response;
464}
465
466
467sub submit_translations
468{
469    # The code of the target language (ensure it is lowercase)
470    my $target_language_code = lc(shift(@_));
471    # The key of the file to translate (ensure it is lowercase)
472    my $translation_file_key = lc(shift(@_));
473    # The username of the translation submitter
474    my $submitter_username = shift(@_);
475    # Whether to submit a target chunk even if it hasn't changed
476    my $force_submission_flag = shift(@_);
477
478    # Check that the necessary arguments were supplied
479    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
480    &log_message("Fatal error (but cannot be thrown): Missing command argument.");
481    die "\n";
482    }
483   
484    my %source_file_key_to_text_mapping = ();
485    my %source_file_key_to_gti_comment_mapping = ();
486    my %target_file_key_to_text_mapping = ();
487    my %target_file_key_to_gti_comment_mapping = ();
488
489    my (@source_file_lines, @target_file_lines) = ();
490    my ($source_file, $target_file, $translation_file_type);
491
492   
493    if ($translation_file_key ne "gs3interface") {
494    # Get (and check) the translation configuration
495    ($source_file, $target_file, $translation_file_type)
496        = &get_translation_configuration($target_language_code, $translation_file_key);
497
498    # Parse the source language and target language files
499    @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
500    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
501    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
502    %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
503
504    @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
505    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
506    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
507    %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
508    }
509    else {
510    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
511                 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
512    }
513    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
514    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
515
516    # Submission date
517    my $day = (localtime)[3];
518    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
519    my $year = (localtime)[5] + 1900;
520    my $submission_date = "$day-$month-$year";
521
522    open(SUBMISSION, "-");
523    my @submission_lines = <SUBMISSION>;
524    close(SUBMISSION);
525
526    # Remove any nasty carriage returns
527    # &log_message("Submission:");
528    foreach my $submission_line (@submission_lines) {
529    $submission_line =~ s/\r$//;
530    #&log_message("  $submission_line");
531    }
532
533    my %source_file_key_to_submission_mapping = ();
534    my %target_file_key_to_submission_mapping = ();
535    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
536    # Read source file part of submission
537    if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
538        my $chunk_key = $1;
539
540        # Read the source file text
541        my $source_file_chunk_text = "";
542        $i++;
543        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
544        $source_file_chunk_text .= $submission_lines[$i];
545        $i++;
546        }
547        $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
548        $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
549
550        #&log_message("Source file key: $chunk_key");
551        #&log_message("Source file text: $source_file_chunk_text");
552        $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
553    }
554
555    # Read target file part of submission
556    if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
557        my $chunk_key = $1;
558
559        # Read the target file text
560        my $target_file_chunk_text = "";
561        $i++;
562        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
563        $target_file_chunk_text .= $submission_lines[$i];
564        $i++;
565        }
566        $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
567        $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
568
569        #&log_message("Target file key: $chunk_key");
570        #&log_message("Target file text: $target_file_chunk_text");
571        $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
572    }
573    }
574
575    # -----------------------------------------
576    #   Validate the translation submissions
577    # -----------------------------------------
578
579    # Check that the translations are valid
580    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
581    # Make sure the submitted chunk still exists in the source file
582    if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
583        &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
584        delete $source_file_key_to_submission_mapping{$chunk_key};
585        delete $target_file_key_to_submission_mapping{$chunk_key};
586        next;
587    }
588
589    # Make sure the submitted source chunk matches the source file chunk
590    if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
591        &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
592        &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
593        &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
594        delete $source_file_key_to_submission_mapping{$chunk_key};
595        delete $target_file_key_to_submission_mapping{$chunk_key};
596        next;
597    }
598    }
599
600    # Apply the submitted translations
601    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
602    # Only apply the submission if it is a change, unless -force_submission has been specified
603    if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
604        $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
605        $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
606    }
607    }
608   
609    if ($translation_file_key ne "gs3interface") {
610    eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
611    } else {
612    eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
613    }
614}
615
616
617sub create_glihelp_zip_file
618{
619    my $target_language_code = shift(@_);
620    my $translation_file_key = "glihelp";
621   
622    &log_message("Creating GLI Help zip file for $target_language_code");
623
624    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
625   
626    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
627    if ( ! -e $classpath) {
628    &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
629    }   
630   
631    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
632    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
633   
634    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
635    if ( ! -e $gen_many_html_xsl_filepath) {
636    &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
637    }
638
639    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
640    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
641   
642    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
643    $target_file_directory = $target_file_directory."/";
644
645    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
646    my $perl_exec = &util::filename_cat($ENV{'PERLPATH'}, "perl");
647    my $java_exec = "java";
648    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
649    $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
650    }
651
652    my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | $perl_exec -S $split_script_filepath $target_file_directory";
653    my $response = `$cmd`;
654    &log_message($cmd);
655    &log_message($response);   
656    &log_message("Created HTML operational files");
657   
658    $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
659    $response = `$cmd`;
660    &log_message($cmd);
661    &log_message($response);   
662    &log_message("Created xml index file");
663
664    my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
665    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
666    $response = `$cmd`;
667    &log_message($cmd);
668    &log_message($response);   
669    &log_message("Created the zip file");
670}
671
672
673sub get_translation_configuration
674{
675    # Get the code of the target language
676    my $target_language_code = shift(@_);
677    # Get the key of the file to translate
678    my $translation_file_key = shift(@_);
679
680    # Read the translation data from the gti.cfg file
681    my ($source_file, $target_file, $translation_file_type) =
682    &get_translation_data_for($target_language_code, $translation_file_key);
683
684    # Check that the file to translate is defined in the gti.cfg file
685    if (!$source_file || !$target_file || !$translation_file_type) {
686    &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
687    }
688
689    # Check that the source file exists
690    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
691    if (!-e $source_file_path) {
692    &throw_fatal_error("Source file $source_file_path does not exist.");
693    }
694
695    # Check that the source file is up to date
696    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
697    # unless ($translation_file_is_not_in_cvs) {
698    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
699        my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
700    if ($source_file_cvs_status =~ /^C /) {
701        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
702    }
703    if ($source_file_cvs_status =~ /^M /) {
704        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
705    }
706    # }
707
708    return ($source_file, $target_file, $translation_file_type);
709}
710
711
712sub get_translation_data_for
713{
714    my ($target_language_code, $translation_file_key) = @_;
715
716    foreach my $translation_file (@$gti_translation_files) {
717    # If this isn't the correct translation file, move onto the next one
718    next if ($translation_file_key ne $translation_file->{'key'});
719
720    # Resolve the target language file
721    my $target_language_file = $translation_file->{'target_file'};
722    if ($target_language_file =~ /(\{.+\;.+\})/) {
723        my $unresolved_target_language_file_part = $1;
724
725        # Check for a special case for the target language code
726        if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
727        my $resolved_target_language_file_part = $2;
728        $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
729        }
730        # Otherwise use the last part as the default value
731        else {
732        my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
733        $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
734        }
735    }
736
737    # Resolve instances of {iso_639_1_target_language_name}
738    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
739    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
740    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
741
742    # Resolve instances of {target_language_code}
743    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
744
745    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
746    }
747
748    return ();
749}
750
751
752sub read_file_lines
753{
754    my ($file_path) = @_;
755
756    if (!open(FILE_IN, "<$file_path")) {
757    &log_message("Note: Could not open file $file_path.");
758    return ();
759    }
760    my @file_lines = <FILE_IN>;
761    close(FILE_IN);
762
763    return @file_lines;
764}
765
766
767sub build_key_to_line_mapping
768{
769    my ($file_lines, $translation_file_type) = @_;
770    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
771}
772
773
774sub build_key_to_text_mapping
775{
776    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
777
778    my %key_to_text_mapping = ();
779    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
780    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
781    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
782
783    my $chunk_text = @$file_lines[$chunk_starting_line];
784    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
785        $chunk_text .= @$file_lines[$l];
786    }
787
788    # Map from chunk key to text
789    eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
790    }
791
792    return %key_to_text_mapping;
793}
794
795
796sub build_key_to_last_update_date_mapping
797{
798    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
799
800    # If the files aren't in CVS then we can't tell anything about what needs updating
801    # return () if ($translation_file_is_not_in_cvs);
802
803    # Build a mapping from key to CVS date
804    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
805    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
806
807    # Build a mapping from key to comment date
808    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
809
810    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
811    my %key_to_last_update_date_mapping = ();
812    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
813    # Use the CVS date as a starting point
814    my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
815    $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
816
817    # If a comment date exists and it is after the CVS date, use that instead
818        # need to convert the comment date format to SVN format
819    my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
820    if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
821        my $chunk_comment_date = $1;           
822        # if the changes were made on 2009 Jan 27, ie. tidying up the GTI commands following each fragment,
823        # then should use the GTI comment date instead
824        # if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
825        if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date)
826         || (&is_date_after($chunk_cvs_date,$chunk_comment_date) && ($chunk_cvs_date eq "2009-01-27" || $chunk_cvs_date eq "2009-01-28")))) {
827        $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
828       
829        # &log_message("cvs date: $chunk_cvs_date, comment date: $chunk_comment_date");
830        # die;
831        }
832    }
833    }
834
835    return %key_to_last_update_date_mapping;
836}
837
838
839sub build_key_to_cvs_date_mapping
840{
841    my ($filename, $translation_file_type) = @_;
842
843    # Use CVS to annotate each line of the file with the date it was last edited
844    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
845    # my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;   
846    # my $cvs_annotated_file = `cd $gsdl_root_directory; export PATH=.:/research/lh92/programs/subversion/bin; svn annotate -v --force $filename`;
847    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v $filename`;
848   
849    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
850
851    my @cvs_annotated_file_lines_date = ();
852    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
853    # Extract the date from the CVS annotation at the front
854        # cvs format : 07-Jun-02
855        # svn format : 2007-07-16
856    # $cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
857        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
858       
859        push(@cvs_annotated_file_lines_date, $1);
860       
861        # trim extra date information in svn annotation format
862        # 15:42:49 +1200 (Wed, 21 Jun 2006)
863        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
864    }   
865   
866    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
867    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
868       
869    my %key_to_cvs_date_mapping = ();
870    foreach my $chunk_key (keys(%key_to_line_mapping)) {
871    my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
872    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
873               
874    # Find the date this chunk was last edited, from the CVS annotation
875    my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
876    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
877        if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
878        # This part of the chunk has been updated more recently
879        $chunk_date = $cvs_annotated_file_lines_date[$l];
880       
881        }
882    }
883
884    # Map from chunk key to CVS date
885    $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
886    }
887
888    return %key_to_cvs_date_mapping;
889}
890
891
892sub build_key_to_gti_comment_mapping
893{
894    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
895
896    my %key_to_gti_comment_mapping = ();
897    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
898    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
899    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
900
901    my $chunk_text = @$file_lines[$chunk_starting_line];
902    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
903        $chunk_text .= @$file_lines[$l];
904    }
905
906    # Map from chunk key to GTI comment
907    my $chunk_gti_comment;
908    eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
909    $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
910    }
911
912    return %key_to_gti_comment_mapping;
913}
914
915
916sub determine_chunks_requiring_translation
917{
918    my $source_file_key_to_text_mapping = shift(@_);
919    my $target_file_key_to_text_mapping = shift(@_);
920
921    # Chunks needing translation are those in the source file with no translation in the target file
922    my @target_file_keys_requiring_translation = ();
923    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
924    if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
925        # &log_message("Chunk with key $chunk_key needs translating.");
926        push(@target_file_keys_requiring_translation, $chunk_key);
927    }
928    }
929
930    return @target_file_keys_requiring_translation;
931}
932
933
934sub determine_chunks_requiring_updating
935{
936    my $source_file_key_to_last_update_date_mapping = shift(@_);
937    my $target_file_key_to_last_update_date_mapping = shift(@_);
938
939    # Chunks needing updating are those in the target file that have been more recently edited in the source file
940    my @target_file_keys_requiring_updating = ();
941    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
942    my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
943    my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
944       
945        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
946                   
947        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
948        # &log_message("Chunk with key $chunk_key needs updating.");
949        push(@target_file_keys_requiring_updating, $chunk_key);
950    }
951    }
952
953    return @target_file_keys_requiring_updating;
954}
955
956
957sub is_chunk_automatically_translated
958{
959    my ($chunk_key, $translation_file_type) = @_;
960    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
961}
962
963
964sub make_text_xml_safe
965{
966    my $text = shift(@_);
967    $text =~ s/\&/\&amp\;/g;
968    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
969    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
970    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
971    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
972    $text =~ s/</\&lt\;/g;
973    $text =~ s/>/\&gt\;/g;
974    return $text;
975}
976
977
978sub unmake_text_xml_safe
979{
980    my $text = shift(@_);
981    $text =~ s/\&lt\;/</g;
982    $text =~ s/\&gt\;/>/g;
983    $text =~ s/\&amp\;/\&/g;
984    return $text;
985}
986
987
988# Returns 1 if $date1 is after $date2, 0 otherwise
989sub is_date_after_cvs
990{
991    my ($date1, $date2) = @_;
992    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
993          "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
994
995    if(!defined $date1) {
996        return 1;
997    }
998
999    my @date1parts = split(/-/, $date1);
1000    my @date2parts = split(/-/, $date2);
1001
1002    # Compare year - nasty because we have rolled over into a new century
1003    my $year1 = $date1parts[2];
1004    if ($year1 < 80) {
1005        $year1 += 2000;
1006    }
1007    my $year2 = $date2parts[2];
1008    if ($year2 < 80) {
1009        $year2 += 2000;
1010    }
1011
1012    # Compare year
1013    if ($year1 > $year2) {
1014    return 1;
1015    }
1016    elsif ($year1 == $year2) {
1017    # Year is the same, so compare month
1018    if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1019        return 1;
1020    }
1021    elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1022        # Month is the same, so compare day
1023        if ($date1parts[0] > $date2parts[0]) {
1024        return 1;
1025        }
1026    }
1027    }
1028
1029    return 0;
1030}
1031
1032sub is_date_after
1033{
1034    my ($date1, $date2) = @_;
1035   
1036    if(!defined $date1) {
1037      return 1;
1038    }
1039    if(!defined $date2) {
1040      return 0;
1041    }
1042   
1043    # 16-Aug-2006
1044    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1045       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1046          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1047       $date1=$3 . "-" . $months{$2} . "-" . $1;
1048       # print "** converted date1: $date1\n";
1049    }
1050    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1051       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1052          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1053       $date2=$3 . "-" . $months{$2} . "-" . $1;
1054       # print "** converted date2: $date2\n";
1055    }
1056   
1057   
1058    # 2006-08-16
1059    my @date1parts = split(/-/, $date1);
1060    my @date2parts = split(/-/, $date2);
1061   
1062    # Compare year
1063    if ($date1parts[0] > $date2parts[0]) {
1064    return 1;
1065    }
1066    elsif ($date1parts[0] == $date2parts[0]) {
1067    # Year is the same, so compare month
1068    if ($date1parts[1] > $date2parts[1]) {
1069        return 1;
1070    }
1071    elsif ($date1parts[1] == $date2parts[1]) {
1072        # Month is the same, so compare day
1073        if ($date1parts[2] > $date2parts[2]) {
1074        return 1;
1075        }
1076    }
1077    }   
1078   
1079    return 0;
1080}
1081
1082
1083sub create_xml_response_for_chunks_requiring_work
1084{
1085    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1086
1087    # Form an XML response to the command
1088    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1089    $xml_response .= "<GTIResponse>\n";
1090    $xml_response .= "  <TranslationFile"
1091    . " key=\"" . $translation_file_key . "\""
1092    . " target_file_path=\"" . $target_file . "\""
1093    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1094    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1095    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1096
1097    # Do chunks requiring translation first
1098    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1099    $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1100    }
1101    else {
1102    $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1103    }
1104
1105    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1106    foreach my $chunk_key (@sorted_chunk_keys) {
1107    last if ($num_chunks_to_return == 0);
1108
1109    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1110    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1111   
1112    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1113    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1114    $xml_response .= "      <TargetFileText></TargetFileText>\n";
1115    $xml_response .= "    </Chunk>\n";
1116
1117    $num_chunks_to_return--;
1118    }
1119
1120    $xml_response .= "  </ChunksRequiringTranslation>\n";
1121
1122    # Then do chunks requiring updating
1123    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1124    $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1125    }
1126    else {
1127    $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1128    }
1129       
1130    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1131    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1132    foreach my $chunk_key (@sorted_chunk_keys) {
1133    last if ($num_chunks_to_return == 0);
1134
1135    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1136    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1137    my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1138    my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1139
1140    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1141    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1142    $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1143    $xml_response .= "    </Chunk>\n";
1144
1145    $num_chunks_to_return--;
1146    }
1147
1148    $xml_response .= "  </ChunksRequiringUpdating>\n";
1149
1150    $xml_response .= "</GTIResponse>\n";
1151
1152    return $xml_response;
1153}
1154
1155
1156sub create_xml_response_for_all_chunks
1157{
1158    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1159
1160    # Form an XML response to the command
1161    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1162    $xml_response .= "<GTIResponse>\n";
1163    $xml_response .= "  <TranslationFile"
1164    . " key=\"" . $translation_file_key . "\""
1165    . " target_file_path=\"" . $target_file . "\"\/>\n";
1166   
1167    # Do all the chunks
1168    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1169
1170    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1171    foreach my $chunk_key (@sorted_chunk_keys) {
1172    my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1173    my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1174
1175    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1176    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1177    if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1178        my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1179        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1180        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1181    }
1182    else {
1183        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1184    }
1185
1186    $xml_response .= "    </Chunk>\n";
1187    }
1188    $xml_response .= "  </Chunks>\n";
1189   
1190    $xml_response .= "</GTIResponse>\n";
1191    return $xml_response;
1192}
1193
1194
1195
1196# ==========================================================================================
1197#   MACROFILE FUNCTIONS
1198
1199sub build_key_to_line_mapping_for_macrofile
1200{
1201    my (@file_lines) = @_;
1202
1203    my $macro_package;
1204    my %chunk_key_to_line_mapping = ();
1205    # Process the contents of the file, line by line
1206    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1207    my $line = $file_lines[$i];
1208    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1209
1210    # Check if a new package is being defined
1211    if ($line =~ m/^package\s+(.+)/) {
1212        $macro_package = $1;
1213    }
1214
1215    # Line contains a macro name
1216    elsif ($line =~ m/^(_\w+_)/) {
1217        my $macro_key = $1;
1218        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1219
1220        # While there is still text of the macro to go...
1221        my $startline = $i;
1222        while ($line !~ /\}$/) {
1223        $i++;
1224        if ($i == scalar(@file_lines)) {
1225            &throw_fatal_error("Could not find end of macro $macro_key.");
1226        }
1227        $line = $file_lines[$i];
1228        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1229        }
1230
1231        # The chunk key consists of the package name and the macro key
1232        my $chunk_key = $macro_package . "." . $macro_key;
1233        # Map from chunk key to line
1234        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1235    }
1236
1237    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1238    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1239        # The chunk key consists of package name and macro key
1240        my $chunk_key = $macro_package . "." . $1;
1241        # Map from chunk key to line
1242        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1243    }
1244    }
1245
1246    return %chunk_key_to_line_mapping;
1247}
1248
1249
1250sub import_chunk_from_macrofile
1251{
1252    my ($chunk_text) = @_;
1253
1254    # Is this an icon macro??
1255    if ($chunk_text =~ /^\#\# (.*)/) {
1256    # Extract image macro text
1257    $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1258    $chunk_text = $1;
1259
1260    # Remove enclosing quotes
1261    $chunk_text =~ s/^\"//;
1262    $chunk_text =~ s/\"$//;
1263    }
1264
1265    # No, so it must be a text macro
1266    else {
1267    # Remove macro key
1268    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1269
1270    # Remove language specifier
1271    $chunk_text =~ s/^\[l=.*\](\s*)//;
1272
1273    # Remove braces enclosing text
1274    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1275    }
1276
1277    return $chunk_text;
1278}
1279
1280
1281sub get_macrofile_chunk_gti_comment
1282{
1283    my ($chunk_text) = @_;
1284
1285    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1286    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1287    return $1;
1288    }
1289
1290    return undef;
1291}
1292
1293
1294sub is_macrofile_chunk_automatically_translated
1295{
1296    my ($chunk_key) = @_;
1297
1298    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1299    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1300    return 1;
1301    }
1302
1303    return 0;
1304}
1305
1306
1307# Use the source file to generate a target file that is formatted the same
1308sub write_translated_macrofile
1309{
1310    my $source_file = shift(@_);  # Not used
1311    my @source_file_lines = @{shift(@_)};
1312    my $source_file_key_to_text_mapping = shift(@_);
1313    my $target_file = shift(@_);
1314    my @target_file_lines = @{shift(@_)};
1315    my $target_file_key_to_text_mapping = shift(@_);
1316    my $target_file_key_to_gti_comment_mapping = shift(@_);
1317    my $target_language_code = shift(@_);
1318
1319    # Build a mapping from source file line to chunk key
1320    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1321    my %source_file_line_to_key_mapping = ();
1322    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1323    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1324    }
1325    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1326    my $source_file_line_number = 0;
1327
1328    # Build a mapping from target file line to chunk key
1329    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1330    my %target_file_line_to_key_mapping = ();
1331    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1332    $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1333    }
1334    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1335
1336    # Write the new target file
1337    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1338    if (!open(TARGET_FILE, ">$target_file_path")) {
1339    &throw_fatal_error("Could not write target file $target_file_path.");
1340    }
1341
1342    # Use the header from the target file, to keep language and author information
1343    if (scalar(@target_file_line_keys) > 0) {
1344    my $target_file_line_number = 0;
1345    my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1346    while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1347        my $target_file_line = $target_file_lines[$target_file_line_number];
1348        last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1349        print TARGET_FILE $target_file_line;
1350        $target_file_line_number++;
1351    }
1352
1353    $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1354    }
1355
1356    # Model the new target file on the source file, with the target file translations
1357    foreach my $line_key (@source_file_line_keys) {
1358    # Fill in the gaps before this chunk starts
1359    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1360    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1361    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1362        print TARGET_FILE $source_file_lines[$source_file_line_number];
1363        $source_file_line_number++;
1364    }
1365    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1366
1367    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1368    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1369    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1370
1371    my $macrofile_key = $chunk_key;
1372    $macrofile_key =~ s/^(.+?)\.//;
1373
1374    # If no translation exists for this chunk, show this, and move on
1375    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1376        print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1377        next;
1378    }
1379
1380    # Grab the source chunk text
1381    my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1382    for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1383        $source_file_chunk .= $source_file_lines[$l];
1384    }
1385
1386    # Is this an icon macro??
1387    if ($source_file_chunk =~ /^\#\# (.*)/) {
1388        # Escape any newline and question mark characters so the source text is replaced correctly
1389        $source_file_chunk_text =~ s/\\/\\\\/g;
1390        $source_file_chunk_text =~ s/\?/\\\?/g;
1391
1392        # Build the new target chunk from the source chunk
1393        my $target_file_chunk = $source_file_chunk;
1394        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1395        $target_file_chunk =~ s/(\s)*$//;
1396        print TARGET_FILE "$target_file_chunk";
1397    }
1398
1399    # No, it is just a normal text macro
1400    else {
1401        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1402    }
1403
1404    # Add the "updated" comment, if one exists
1405    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1406        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1407    }
1408    print TARGET_FILE "\n";
1409    }
1410
1411    close(TARGET_FILE);
1412}
1413
1414
1415sub sort_by_line
1416{
1417    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1418}
1419
1420
1421# ==========================================================================================
1422#   RESOURCE BUNDLE FUNCTIONS
1423
1424sub build_key_to_line_mapping_for_resource_bundle
1425{
1426    my (@file_lines) = @_;
1427
1428    my %chunk_key_to_line_mapping = ();
1429    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1430    my $line = $file_lines[$i];
1431    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1432
1433    # Line contains a dictionary string
1434    if ($line =~ /^(\S+?)[:|=](.*)$/) {
1435        my $chunk_key = $1;
1436
1437        # Map from chunk key to line
1438        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1439    }
1440    }
1441
1442    return %chunk_key_to_line_mapping;
1443}
1444
1445
1446sub import_chunk_from_resource_bundle
1447{
1448    my ($chunk_text) = @_;
1449
1450    # Simple: just remove string key
1451    $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1452    $chunk_text =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1453    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1454
1455    return $chunk_text;
1456}
1457
1458
1459sub get_resource_bundle_chunk_gti_comment
1460{
1461    my ($chunk_text) = @_;
1462
1463    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1464    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1465    return $1;
1466    }
1467
1468    return undef;
1469}
1470
1471
1472sub is_resource_bundle_chunk_automatically_translated
1473{
1474    # No resource bundle chunks are automatically translated
1475    return 0;
1476}
1477
1478
1479sub write_translated_resource_bundle
1480{
1481    my $source_file = shift(@_);  # Not used
1482    my @source_file_lines = @{shift(@_)};
1483    my $source_file_key_to_text_mapping = shift(@_);
1484    my $target_file = shift(@_);
1485    my @target_file_lines = @{shift(@_)};  # Not used
1486    my $target_file_key_to_text_mapping = shift(@_);
1487    my $target_file_key_to_gti_comment_mapping = shift(@_);
1488    my $target_language_code = shift(@_);  # Not used
1489
1490    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1491    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1492    my %source_file_line_to_key_mapping = ();
1493    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1494    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1495    }
1496
1497    # Write the new target file
1498    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1499    if (!open(TARGET_FILE, ">$target_file_path")) {
1500    &throw_fatal_error("Could not write target file $target_file_path.");
1501    }
1502
1503    # Model the new target file on the source file, with the target file translations
1504    my $source_file_line_number = 0;
1505    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1506    # Fill in the gaps before this chunk starts
1507    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1508    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1509    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1510        print TARGET_FILE $source_file_lines[$source_file_line_number];
1511        $source_file_line_number++;
1512    }
1513    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1514
1515    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1516    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1517    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1518
1519    # If no translation exists for this chunk, show this, and move on
1520    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1521        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1522        next;
1523    }
1524
1525    print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1526    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1527        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1528    }
1529    print TARGET_FILE "\n";
1530    }
1531
1532    close(TARGET_FILE);
1533}
1534
1535
1536# ==========================================================================================
1537#   GREENSTONE XML FUNCTIONS
1538
1539sub build_key_to_line_mapping_for_greenstone_xml
1540{
1541    my (@file_lines) = @_;
1542
1543    my %chunk_key_to_line_mapping = ();
1544    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1545    my $line = $file_lines[$i];
1546    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1547
1548    # Line contains a string to translate
1549    if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1550        my $chunk_key = $1;
1551        $line =~ s/\s*$//;  # Remove any nasty whitespace
1552        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1553
1554        # While there is still text of the string to go...
1555        my $startline = $i;
1556        while ($line !~ /<\/Text>$/) {
1557        $i++;
1558        if ($i == scalar(@file_lines)) {
1559            &throw_fatal_error("Could not find end of string $chunk_key.");
1560        }
1561        $line = $file_lines[$i];
1562        $line =~ s/\s*$//;  # Remove any nasty whitespace
1563        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1564        }
1565
1566        # Map from chunk key to line
1567        if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1568        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1569        }
1570        else {
1571        &throw_fatal_error("Duplicate key $chunk_key.");
1572        }
1573    }
1574    }
1575
1576    return %chunk_key_to_line_mapping;
1577}
1578
1579
1580sub import_chunk_from_greenstone_xml
1581{
1582    my ($chunk_text) = @_;
1583
1584    # Simple: just remove the Text tags
1585    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1586    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1587    $chunk_text =~ s/<\/Text>$//;
1588
1589    return $chunk_text;
1590}
1591
1592
1593sub get_greenstone_xml_chunk_gti_comment
1594{
1595    my ($chunk_text) = @_;
1596
1597    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1598    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1599    return $1;
1600    }
1601
1602    return undef;
1603}
1604
1605
1606sub is_greenstone_xml_chunk_automatically_translated
1607{
1608    # No greenstone XML chunks are automatically translated
1609    return 0;
1610}
1611
1612
1613sub write_translated_greenstone_xml
1614{
1615    my $source_file = shift(@_);  # Not used
1616    my @source_file_lines = @{shift(@_)};
1617    my $source_file_key_to_text_mapping = shift(@_);
1618    my $target_file = shift(@_);
1619    my @target_file_lines = @{shift(@_)};  # Not used
1620    my $target_file_key_to_text_mapping = shift(@_);
1621    my $target_file_key_to_gti_comment_mapping = shift(@_);
1622    my $target_language_code = shift(@_);  # Not used
1623
1624    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1625    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1626    my %source_file_line_to_key_mapping = ();
1627    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1628    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1629    }
1630
1631    # Write the new target file
1632    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1633    if (!open(TARGET_FILE, ">$target_file_path")) {
1634    &throw_fatal_error("Could not write target file $target_file_path.");
1635    }
1636
1637    # Model the new target file on the source file, with the target file translations
1638    my $source_file_line_number = 0;
1639    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1640    # Fill in the gaps before this chunk starts
1641    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1642    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1643    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1644        print TARGET_FILE $source_file_lines[$source_file_line_number];
1645        $source_file_line_number++;
1646    }
1647    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1648
1649    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1650    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1651    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1652    $target_file_chunk_text =~ s/(\n)*$//g;
1653
1654    # If no translation exists for this chunk, show this, and move on
1655    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1656        print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1657        next;
1658    }
1659
1660    print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1661    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1662        my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1663        $chunk_gti_comment =~ s/^Updated //;
1664        print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1665    }
1666    print TARGET_FILE "\n";
1667    }
1668
1669    # Fill in the end of the file
1670    while ($source_file_line_number < scalar(@source_file_lines)) {
1671    print TARGET_FILE $source_file_lines[$source_file_line_number];
1672    $source_file_line_number++;
1673    }
1674
1675    close(TARGET_FILE);
1676}
1677
1678
1679# ==========================================================================================
1680#   GREENSTONE3 FUNCTIONS
1681
1682sub get_all_chunks_for_gs3
1683{
1684    # The code of the target language (ensure it is lowercase)
1685    my $target_language_code = lc(shift(@_));
1686    my $translation_file_key = lc(shift(@_));
1687
1688    # Check that the necessary arguments were supplied
1689    if (!$target_language_code) {
1690    &throw_fatal_error("Missing command argument.");
1691    }
1692
1693    # Get (and check) the translation configuration
1694    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1695   
1696    my %source_files_key_to_text_mapping = ();
1697    my %target_files_key_to_text_mapping = ();
1698    my %source_files_key_to_last_update_date_mapping = ();
1699    my %target_files_key_to_last_update_date_mapping = ();
1700
1701    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1702
1703    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1704    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1705
1706    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1707    return $xml_response;
1708}
1709
1710
1711sub get_first_n_chunks_requiring_work_for_gs3
1712{
1713    # The code of the target language (ensure it is lowercase)
1714    my $target_language_code = lc(shift(@_));
1715    # The key of the file to translate (ensure it is lowercase)
1716    my $translation_file_key = lc(shift(@_));
1717    # The number of chunks to return (defaults to one if not specified)
1718    my $num_chunks_to_return = shift(@_) || "1";
1719   
1720    # Check that the necessary arguments were supplied
1721    if (!$target_language_code || !$translation_file_key) {
1722    &throw_fatal_error("Missing command argument.");
1723    }
1724   
1725    my %source_files_key_to_text_mapping = ();
1726    my %target_files_key_to_text_mapping = ();
1727    my %source_files_key_to_last_update_date_mapping = ();
1728    my %target_files_key_to_last_update_date_mapping = ();
1729
1730    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1731                 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1732   
1733    # Determine the target file chunks requiring translation
1734    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1735    # Determine the target file chunks requiring updating
1736    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1737    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1738    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1739
1740    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1741                                      \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1742                                      $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1743                                      \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1744   
1745    return $xml_response;
1746}
1747
1748
1749
1750sub build_gs3_configuration
1751{
1752    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1753    $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1754   
1755    my $source_file_directory = "greenstone3";
1756    my $translation_file_type = "resource_bundle";
1757
1758    foreach my $interface_file_key (@gs3_interface_files) {
1759
1760    &log_message("Greenstone 3 interface file: " . $interface_file_key);
1761
1762    # Parse the source language and target language files
1763    my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1764    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1765    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1766    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1767    my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1768   
1769    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1770    my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1771    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1772    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1773    my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1774
1775
1776    # Filter out any automatically translated chunks
1777    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1778        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1779        delete $source_file_key_to_line_mapping{$chunk_key};
1780        delete $target_file_key_to_line_mapping{$chunk_key};
1781        }
1782    }
1783
1784    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1785    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1786
1787    foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1788        my $global_chunk_key = "$interface_file_key.$chunk_key";
1789        $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1790        $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1791
1792        if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1793        $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1794        $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1795        }
1796    }   
1797    }
1798}
1799
1800
1801sub write_translated_gs3interface
1802{
1803    my $source_file_key_to_text_mapping = shift(@_);
1804    my $target_file_key_to_text_mapping = shift(@_);
1805    my $target_file_key_to_gti_comment_mapping = shift(@_);
1806    my $target_language_code = shift(@_);
1807   
1808    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1809
1810    my %translated_interface_file_keys = ();
1811    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1812    $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1813    if (!defined $translated_interface_file_keys{$1}) {
1814        &log_message("Updated interface file: " . $1); 
1815        $translated_interface_file_keys{$1}="";
1816    }
1817    }
1818    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1819   
1820    my $source_file_directory = "greenstone3";   
1821   
1822    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1823   
1824    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1825    my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1826    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1827    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1828    my %source_file_line_to_key_mapping = ();
1829    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1830        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1831    }
1832   
1833    # Write the new target file
1834    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1835    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1836    if (!open(TARGET_FILE, ">$target_file_path")) {
1837        &throw_fatal_error("Could not write target file $target_file_path.");
1838    }
1839
1840    # Model the new target file on the source file, with the target file translations
1841    my $source_file_line_number = 0;
1842    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1843        # Fill in the gaps before this chunk starts
1844        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1845        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1846        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1847        print TARGET_FILE $source_file_lines[$source_file_line_number];
1848        $source_file_line_number++;
1849        }
1850        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1851
1852        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1853        my $global_chunk_key = "$interface_file_key.$chunk_key";
1854        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1855        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1856
1857        # If no translation exists for this chunk, show this, and move on
1858        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1859        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1860        next;
1861        }
1862
1863        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1864        if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1865        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1866        }
1867        print TARGET_FILE "\n";
1868    }
1869
1870    close(TARGET_FILE);
1871    }           
1872}
1873
1874&main(@ARGV);
Note: See TracBrowser for help on using the browser.