root/main/trunk/greenstone2/bin/script/gti.pl @ 24362

Revision 24362, 76.9 KB (checked in by ak19, 8 years ago)

The method of locating perl has changed once more: util now defines the fuction get_perl_exec which is used by other scripts to obtain the path to the perl executable they should use.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35use iso639;
36use strict;
37use util;
38
39
40#my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
41#my $anonymous_svn_root = "http://http://svn.greenstone.org/gsdl/trunk/";
42my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
43my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
44my $source_language_code = "en";  # This is non-negiotable
45
46my $gti_translation_files =
47    [ # Greenstone macrofiles
48      { 'key' => "coredm",
49    'file_type' => "macrofile",
50    'source_file' => "macros/english.dm",
51    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
52      { 'key' => "auxdm",
53    'file_type' => "macrofile",
54    'source_file' => "macros/english2.dm",
55    'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57      # GLI dictionary
58      { 'key' => "glidict",
59    'file_type' => "resource_bundle",
60    'source_file' => "gli/classes/dictionary.properties",
61    'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
62
63      # GLI help
64      { 'key' => "glihelp",
65    'file_type' => "greenstone_xml",
66    'source_file' => "gli/help/en/help.xml",
67    'target_file' => "gli/help/{target_language_code}/help.xml" },
68
69      # Greenstone Perl modules
70      { 'key' => "perlmodules",
71    'file_type' => "resource_bundle",
72    'source_file' => "perllib/strings.properties",
73    'target_file' => "perllib/strings_{target_language_code}.properties" },
74
75      # Greenstone tutorial exercises
76      # { 'key' => "tutorials",
77    # 'file_type' => "greenstone_xml",
78    # 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
79    # 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
80
81      # new Greenstone.org
82      { 'key' => "greenorg",
83        'file_type' => "resource_bundle",
84        'source_file' => "greenstoneorg/website/classes/Gsc.properties",
85        'target_file' => "greenstoneorg/website/classes/Gsc_{iso_639_1_target_language_name}.properties"
86    # 'file_type' => "macrofile",
87    # 'source_file' => "greenorg/macros/english.dm",
88    # 'target_file' => "greenorg/macros/{iso_639_1_target_language_name}.dm"
89      }
90
91      # { 'key' => "gs3interface",
92    #'file_type' => "resource_bundle",
93    #'source_file' => "greenstone3",
94    #'target_file' => "greenstone3"
95      #}
96    ];
97
98my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
99
100
101sub main
102{
103    # Get the command to process, and any arguments
104    my $gti_command = shift(@_);
105    my @gti_command_arguments = @_;
106    my $module = $_[1];
107
108    # Open the GTI log file for appending, or write to STDERR if that fails
109    if (!open(GTI_LOG, ">>$gti_log_file")) {
110    open(GTI_LOG, ">&STDERR");
111    }
112
113    # Log the command that launched this script
114    &log_message("Command: $0 @ARGV");
115
116    # Check that a command was supplied
117    if (!$gti_command) {
118    &throw_fatal_error("Missing command.");
119    }     
120
121    # Process the command
122    if ($gti_command =~ /^get-all-chunks$/i) {
123    # Check that GS3 interface is the target
124    if ($module eq "gs3interface") {       
125        print &get_all_chunks_for_gs3(@gti_command_arguments);
126    } else {
127        print &get_all_chunks(@gti_command_arguments);
128    }
129    }
130    if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
131    if ($module eq "gs3interface") {       
132        print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
133    } else {
134        print &get_first_n_chunks_requiring_work(@gti_command_arguments);
135    }
136    }
137    if ($gti_command =~ /^get-language-status$/i) {
138    print &get_language_status(@gti_command_arguments);       
139    }
140    if ($gti_command =~ /^search-chunks$/i) {
141    print &search_chunks(@gti_command_arguments);
142    }
143    if ($gti_command =~ /^submit-translations$/i) {
144    # This command cannot produce any output since it reads input
145    &submit_translations(@gti_command_arguments);
146    }
147    if ($gti_command =~ /^create-glihelp-zip-file$/i) {
148    # This command cannot produce any output since it reads input
149    &create_glihelp_zip_file(@gti_command_arguments);
150    }
151#    else {
152    # The command was not recognized
153    #&throw_fatal_error("Unknown command \"$gti_command\".");
154    #}
155}
156
157
158sub throw_fatal_error
159{
160    my $error_message = shift(@_);
161
162    # Write an XML error response
163    print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
164    print "<GTIResponse>\n";
165    print "  <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
166    print "</GTIResponse>\n";
167
168    # Log the error message, then die
169    &log_message("Error: $error_message");
170    die "\n";
171}
172
173
174sub log_message
175{
176    my $log_message = shift(@_);
177    print GTI_LOG time() . " -- " . $log_message . "\n";
178}
179
180
181sub get_all_chunks
182{
183    # The code of the target language (ensure it is lowercase)
184    my $target_language_code = lc(shift(@_));
185    # The key of the file to translate (ensure it is lowercase)
186    my $translation_file_key = lc(shift(@_));
187
188    # Check that the necessary arguments were supplied
189    if (!$target_language_code || !$translation_file_key) {
190    &throw_fatal_error("Missing command argument.");
191    }
192
193    # Get (and check) the translation configuration
194    my ($source_file, $target_file, $translation_file_type)
195    = &get_translation_configuration($target_language_code, $translation_file_key);
196
197    # Parse the source language and target language files
198    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
199    my @source_file_lines = &read_file_lines($source_file_path);
200    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
201   
202    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
203    my @target_file_lines = &read_file_lines($target_file_path);
204    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
205
206    # Filter out any automatically translated chunks
207    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
208    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
209        delete $source_file_key_to_line_mapping{$chunk_key};
210        delete $target_file_key_to_line_mapping{$chunk_key};
211    }
212    }
213
214    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
215    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
216    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
217    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
218
219    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
220    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
221
222    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
223   
224    return $xml_response;
225}
226
227
228sub get_first_n_chunks_requiring_work
229{
230    # The code of the target language (ensure it is lowercase)
231    my $target_language_code = lc(shift(@_));
232    # The key of the file to translate (ensure it is lowercase)
233    my $translation_file_key = lc(shift(@_));
234    # The number of chunks to return (defaults to one if not specified)
235    my $num_chunks_to_return = shift(@_) || "1";
236
237    # Check that the necessary arguments were supplied
238    if (!$target_language_code || !$translation_file_key) {
239    &throw_fatal_error("Missing command argument.");
240    }
241
242    # Get (and check) the translation configuration
243    my ($source_file, $target_file, $translation_file_type)
244    = &get_translation_configuration($target_language_code, $translation_file_key);
245
246    # Parse the source language and target language files
247    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
248    my @source_file_lines = &read_file_lines($source_file_path);
249    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
250   
251    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
252    my @target_file_lines = &read_file_lines($target_file_path);
253    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
254
255    # Filter out any automatically translated chunks
256    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
257    if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
258        delete $source_file_key_to_line_mapping{$chunk_key};
259        delete $target_file_key_to_line_mapping{$chunk_key};
260    }
261    }
262
263    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
264    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
265    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
266    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
267
268    # Determine the target file chunks requiring translation
269    my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
270    &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
271
272    # Determine the target file chunks requiring updating
273    my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
274    my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
275    my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
276    &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
277
278    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);   
279   
280    return $xml_response;
281}
282
283
284sub get_language_status
285{
286    # The code of the target language (ensure it is lowercase)
287    my $target_language_code = lc(shift(@_));
288
289    # Check that the necessary arguments were supplied
290    if (!$target_language_code) {
291    &throw_fatal_error("Missing command argument.");
292    }
293
294    # Form an XML response to the command
295    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
296    $xml_response .= "<GTIResponse>\n";
297    $xml_response .= "  <LanguageStatus code=\"$target_language_code\">\n";
298
299    foreach my $translation_file (@$gti_translation_files) {   
300    my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
301    my $target_file_name = "";
302   
303    if ($translation_file->{'key'} eq "gs3interface") {
304        my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
305        &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );   
306       
307        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
308        my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
309
310        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
311        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
312        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
313        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
314    }
315    else {
316        # Get (and check) the translation configuration
317        my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
318        $target_file_name = $target_file;
319
320        # Parse the source language and target language files
321        my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
322        my @source_file_lines = &read_file_lines($source_file_path);
323        my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
324
325        my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
326        my @target_file_lines = &read_file_lines($target_file_path);
327        my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
328
329        # Filter out any automatically translated chunks
330        foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
331        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
332            delete $source_file_key_to_line_mapping{$chunk_key};
333            delete $target_file_key_to_line_mapping{$chunk_key};
334        }
335        }
336
337        my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
338        my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
339
340        # Determine the target file chunks requiring translation
341        my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);     
342
343        # Determine the target file chunks requiring updating
344        my @target_file_keys_requiring_updating = ();
345        if (-e $target_file_path) {
346        my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
347        my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
348        @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);     
349        }
350
351        $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
352        $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
353        $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
354        $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
355    }
356   
357    &log_message("Status of " . $translation_file->{'key'});
358    &log_message("Number of source chunks: " . $num_source_chunks);
359    &log_message("Number of target chunks: " . $num_target_chunks);
360    &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
361    &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
362
363    $xml_response .= "    <TranslationFile"
364        . " key=\"" . $translation_file->{'key'} . "\""
365        . " target_file_path=\"" . $target_file_name . "\""
366        . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
367        . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
368        . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
369    }
370
371    $xml_response .= "  </LanguageStatus>\n";
372
373    $xml_response .= "</GTIResponse>\n";
374    return $xml_response;
375}
376
377
378sub search_chunks
379{
380    # The code of the target language (ensure it is lowercase)
381    my $target_language_code = lc(shift(@_));
382    # The key of the file to translate (ensure it is lowercase)
383    my $translation_file_key = lc(shift(@_));
384    # The query string
385    my $query_string = join(' ', @_);
386
387    # Check that the necessary arguments were supplied
388    if (!$target_language_code || !$translation_file_key || !$query_string) {
389    &throw_fatal_error("Missing command argument.");
390    }
391
392    my ($source_file, $target_file, $translation_file_type) = ();
393    my %source_file_key_to_text_mapping = ();
394    my %target_file_key_to_text_mapping = ();
395   
396   
397    if ($translation_file_key ne "gs3interface") {
398    # Get (and check) the translation configuration
399    ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
400
401    # Parse the source language and target language files
402    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
403    my @source_file_lines = &read_file_lines($source_file_path);
404    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
405
406    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
407    my @target_file_lines = &read_file_lines($target_file_path);
408    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
409
410    # Filter out any automatically translated chunks
411    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
412        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
413        delete $source_file_key_to_line_mapping{$chunk_key};
414        delete $target_file_key_to_line_mapping{$chunk_key};
415        }
416    }
417
418    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
419    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
420    }
421    else {
422    # Not needed in this case
423    my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
424    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
425                 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
426    }
427
428    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
429    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
430
431    # Determine the target file chunks matching the query
432    my @target_file_keys_matching_query = ();
433    foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
434    my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
435    if ($target_file_text =~ /$query_string/i) {
436        # &log_message("Chunk with key $chunk_key matches query.");
437        push(@target_file_keys_matching_query, $chunk_key);
438    }
439    }
440
441    # Form an XML response to the command
442    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
443    $xml_response .= "<GTIResponse>\n";
444
445    $xml_response .= "  <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
446    foreach my $chunk_key (@target_file_keys_matching_query) {
447    my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
448
449    $xml_response .= "    <Chunk key=\"$chunk_key\">\n";
450    $xml_response .= "      <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
451    $xml_response .= "    </Chunk>\n";
452    }
453    $xml_response .= "  </ChunksMatchingQuery>\n";
454
455    $xml_response .= "</GTIResponse>\n";
456    return $xml_response;
457}
458
459
460sub submit_translations
461{
462    # The code of the target language (ensure it is lowercase)
463    my $target_language_code = lc(shift(@_));
464    # The key of the file to translate (ensure it is lowercase)
465    my $translation_file_key = lc(shift(@_));
466    # The username of the translation submitter
467    my $submitter_username = shift(@_);
468    # Whether to submit a target chunk even if it hasn't changed
469    my $force_submission_flag = shift(@_);
470
471    # Check that the necessary arguments were supplied
472    if (!$target_language_code || !$translation_file_key || !$submitter_username) {
473    &log_message("Fatal error (but cannot be thrown): Missing command argument.");
474    die "\n";
475    }
476   
477    my %source_file_key_to_text_mapping = ();
478    my %source_file_key_to_gti_comment_mapping = ();
479    my %target_file_key_to_text_mapping = ();
480    my %target_file_key_to_gti_comment_mapping = ();
481
482    my (@source_file_lines, @target_file_lines) = ();
483    my ($source_file, $target_file, $translation_file_type);
484
485   
486    if ($translation_file_key ne "gs3interface") {
487    # Get (and check) the translation configuration
488    ($source_file, $target_file, $translation_file_type)
489        = &get_translation_configuration($target_language_code, $translation_file_key);
490
491    # Parse the source language and target language files
492    @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
493    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
494    %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
495    %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);   
496
497    @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
498    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
499    %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
500    %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);   
501    }
502    else {
503    &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
504                 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
505    }
506    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
507    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
508
509    # Submission date
510    my $day = (localtime)[3];
511    my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
512    my $year = (localtime)[5] + 1900;
513    my $submission_date = "$day-$month-$year";
514
515    open(SUBMISSION, "-");
516    my @submission_lines = <SUBMISSION>;
517    close(SUBMISSION);
518
519    # Remove any nasty carriage returns
520    # &log_message("Submission:");
521    foreach my $submission_line (@submission_lines) {
522    $submission_line =~ s/\r$//;
523    #&log_message("  $submission_line");
524    }
525
526    my %source_file_key_to_submission_mapping = ();
527    my %target_file_key_to_submission_mapping = ();
528    for (my $i = 0; $i < scalar(@submission_lines); $i++) {
529    # Read source file part of submission
530    if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
531        my $chunk_key = $1;
532
533        # Read the source file text
534        my $source_file_chunk_text = "";
535        $i++;
536        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
537        $source_file_chunk_text .= $submission_lines[$i];
538        $i++;
539        }
540        $source_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
541        $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
542
543        #&log_message("Source file key: $chunk_key");
544        #&log_message("Source file text: $source_file_chunk_text");
545        $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
546    }
547
548    # Read target file part of submission
549    if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
550        my $chunk_key = $1;
551
552        # Read the target file text
553        my $target_file_chunk_text = "";
554        $i++;
555        while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
556        $target_file_chunk_text .= $submission_lines[$i];
557        $i++;
558        }
559        $target_file_chunk_text =~ s/\n$//;  # Strip the extra newline character added
560        $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
561
562        #&log_message("Target file key: $chunk_key");
563        #&log_message("Target file text: $target_file_chunk_text");
564        $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
565    }
566    }
567
568    # -----------------------------------------
569    #   Validate the translation submissions
570    # -----------------------------------------
571
572    # Check that the translations are valid
573    foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
574    # Make sure the submitted chunk still exists in the source file
575    if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
576        &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
577        delete $source_file_key_to_submission_mapping{$chunk_key};
578        delete $target_file_key_to_submission_mapping{$chunk_key};
579        next;
580    }
581
582    # Make sure the submitted source chunk matches the source file chunk
583    if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
584        &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
585        &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
586        &log_message("      Source text: $source_file_key_to_text_mapping{$chunk_key}");
587        delete $source_file_key_to_submission_mapping{$chunk_key};
588        delete $target_file_key_to_submission_mapping{$chunk_key};
589        next;
590    }
591    }
592
593    # Apply the submitted translations
594    foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
595    # Only apply the submission if it is a change, unless -force_submission has been specified
596    if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
597        $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
598        $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
599    }
600    }
601   
602    if ($translation_file_key ne "gs3interface") {
603    eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
604    } else {
605    eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
606    }
607}
608
609
610sub create_glihelp_zip_file
611{
612    my $target_language_code = shift(@_);
613    my $translation_file_key = "glihelp";
614   
615    &log_message("Creating GLI Help zip file for $target_language_code");
616
617    my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);   
618   
619    my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
620    if ( ! -e $classpath) {
621    &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
622    }   
623   
624    my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
625    $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
626   
627    my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
628    if ( ! -e $gen_many_html_xsl_filepath) {
629    &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
630    }
631
632    my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");   
633    my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");   
634   
635    my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
636    $target_file_directory = $target_file_directory."/";
637
638    my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
639    my $perl_exec = &util::get_perl_exec();
640    my $java_exec = "java";
641    if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
642    $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
643    }
644
645    my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
646    my $response = `$cmd`;
647    &log_message($cmd);
648    &log_message($response);   
649    &log_message("Created HTML operational files");
650   
651    $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
652    $response = `$cmd`;
653    &log_message($cmd);
654    &log_message($response);   
655    &log_message("Created xml index file");
656
657    my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
658    $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
659    $response = `$cmd`;
660    &log_message($cmd);
661    &log_message($response);   
662    &log_message("Created the zip file");
663}
664
665
666sub get_translation_configuration
667{
668    # Get the code of the target language
669    my $target_language_code = shift(@_);
670    # Get the key of the file to translate
671    my $translation_file_key = shift(@_);
672
673    # Read the translation data from the gti.cfg file
674    my ($source_file, $target_file, $translation_file_type) =
675    &get_translation_data_for($target_language_code, $translation_file_key);
676
677    # Check that the file to translate is defined in the gti.cfg file
678    if (!$source_file || !$target_file || !$translation_file_type) {
679    &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
680    }
681
682    # Check that the source file exists
683    my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
684    if (!-e $source_file_path) {
685    &throw_fatal_error("Source file $source_file_path does not exist.");
686    }
687
688    # Check that the source file is up to date
689    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
690    # unless ($translation_file_is_not_in_cvs) {
691    #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
692        my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
693    if ($source_file_cvs_status =~ /^C /) {
694        &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
695    }
696    if ($source_file_cvs_status =~ /^M /) {
697        &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
698    }
699    # }
700
701    return ($source_file, $target_file, $translation_file_type);
702}
703
704
705sub get_translation_data_for
706{
707    my ($target_language_code, $translation_file_key) = @_;
708
709    foreach my $translation_file (@$gti_translation_files) {
710    # If this isn't the correct translation file, move onto the next one
711    next if ($translation_file_key ne $translation_file->{'key'});
712
713    # Resolve the target language file
714    my $target_language_file = $translation_file->{'target_file'};
715    if ($target_language_file =~ /(\{.+\;.+\})/) {
716        my $unresolved_target_language_file_part = $1;
717
718        # Check for a special case for the target language code
719        if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
720        my $resolved_target_language_file_part = $2;
721        $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
722        }
723        # Otherwise use the last part as the default value
724        else {
725        my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
726        $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;           
727        }
728    }
729
730    # Resolve instances of {iso_639_1_target_language_name}
731    my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
732    $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
733    $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
734
735    # Resolve instances of {target_language_code}
736    $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
737
738    return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
739    }
740
741    return ();
742}
743
744
745sub read_file_lines
746{
747    my ($file_path) = @_;
748
749    if (!open(FILE_IN, "<$file_path")) {
750    &log_message("Note: Could not open file $file_path.");
751    return ();
752    }
753    my @file_lines = <FILE_IN>;
754    close(FILE_IN);
755
756    return @file_lines;
757}
758
759
760sub build_key_to_line_mapping
761{
762    my ($file_lines, $translation_file_type) = @_;
763    eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
764}
765
766
767sub build_key_to_text_mapping
768{
769    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
770
771    my %key_to_text_mapping = ();
772    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
773    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
774    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
775
776    my $chunk_text = @$file_lines[$chunk_starting_line];
777    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
778        $chunk_text .= @$file_lines[$l];
779    }
780
781    # Map from chunk key to text
782    eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
783    }
784
785    return %key_to_text_mapping;
786}
787
788
789sub build_key_to_last_update_date_mapping
790{
791    my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
792
793    # If the files aren't in CVS then we can't tell anything about what needs updating
794    # return () if ($translation_file_is_not_in_cvs);
795
796    # Build a mapping from key to CVS date
797    # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
798    my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
799
800    # Build a mapping from key to comment date
801    my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
802
803    # Build a mapping from key to last update date (the latter of the CVS date and comment date)
804    my %key_to_last_update_date_mapping = ();
805    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
806    # Use the CVS date as a starting point
807    my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
808    $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
809
810    # If a comment date exists and it is after the CVS date, use that instead
811        # need to convert the comment date format to SVN format
812    my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
813    if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
814        my $chunk_comment_date = $1;           
815        # if the changes were made on 2009 Jan 27, ie. tidying up the GTI commands following each fragment,
816        # then should use the GTI comment date instead
817        # if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
818        if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date)
819         || (&is_date_after($chunk_cvs_date,$chunk_comment_date) && ($chunk_cvs_date eq "2009-01-27" || $chunk_cvs_date eq "2009-01-28")))) {
820        $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
821       
822        # &log_message("cvs date: $chunk_cvs_date, comment date: $chunk_comment_date");
823        # die;
824        }
825    }
826    }
827
828    return %key_to_last_update_date_mapping;
829}
830
831
832sub build_key_to_cvs_date_mapping
833{
834    my ($filename, $translation_file_type) = @_;
835
836    # Use CVS to annotate each line of the file with the date it was last edited
837    # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
838    # my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;   
839    # my $cvs_annotated_file = `cd $gsdl_root_directory; export PATH=.:/research/lh92/programs/subversion/bin; svn annotate -v --force $filename`;
840    my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v $filename`;
841   
842    my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
843
844    my @cvs_annotated_file_lines_date = ();
845    foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
846    # Extract the date from the CVS annotation at the front
847        # cvs format : 07-Jun-02
848        # svn format : 2007-07-16
849    # $cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
850        $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
851       
852        push(@cvs_annotated_file_lines_date, $1);
853       
854        # trim extra date information in svn annotation format
855        # 15:42:49 +1200 (Wed, 21 Jun 2006)
856        $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
857    }   
858   
859    # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
860    my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
861       
862    my %key_to_cvs_date_mapping = ();
863    foreach my $chunk_key (keys(%key_to_line_mapping)) {
864    my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
865    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
866               
867    # Find the date this chunk was last edited, from the CVS annotation
868    my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];       
869    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
870        if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
871        # This part of the chunk has been updated more recently
872        $chunk_date = $cvs_annotated_file_lines_date[$l];
873       
874        }
875    }
876
877    # Map from chunk key to CVS date
878    $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
879    }
880
881    return %key_to_cvs_date_mapping;
882}
883
884
885sub build_key_to_gti_comment_mapping
886{
887    my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
888
889    my %key_to_gti_comment_mapping = ();
890    foreach my $chunk_key (keys(%$key_to_line_mapping)) {
891    my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
892    my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
893
894    my $chunk_text = @$file_lines[$chunk_starting_line];
895    for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
896        $chunk_text .= @$file_lines[$l];
897    }
898
899    # Map from chunk key to GTI comment
900    my $chunk_gti_comment;
901    eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
902    $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
903    }
904
905    return %key_to_gti_comment_mapping;
906}
907
908
909sub determine_chunks_requiring_translation
910{
911    my $source_file_key_to_text_mapping = shift(@_);
912    my $target_file_key_to_text_mapping = shift(@_);
913
914    # Chunks needing translation are those in the source file with no translation in the target file
915    my @target_file_keys_requiring_translation = ();
916    foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
917    if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
918        # &log_message("Chunk with key $chunk_key needs translating.");
919        push(@target_file_keys_requiring_translation, $chunk_key);
920    }
921    }
922
923    return @target_file_keys_requiring_translation;
924}
925
926
927sub determine_chunks_requiring_updating
928{
929    my $source_file_key_to_last_update_date_mapping = shift(@_);
930    my $target_file_key_to_last_update_date_mapping = shift(@_);
931
932    # Chunks needing updating are those in the target file that have been more recently edited in the source file
933    my @target_file_keys_requiring_updating = ();
934    foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
935    my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
936    my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
937       
938        # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";       
939                   
940        if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
941        # &log_message("Chunk with key $chunk_key needs updating.");
942        push(@target_file_keys_requiring_updating, $chunk_key);
943    }
944    }
945
946    return @target_file_keys_requiring_updating;
947}
948
949
950sub is_chunk_automatically_translated
951{
952    my ($chunk_key, $translation_file_type) = @_;
953    eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
954}
955
956
957sub make_text_xml_safe
958{
959    my $text = shift(@_);
960    $text =~ s/\&/\&amp\;/g;
961    $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
962    $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
963    $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
964    $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
965    $text =~ s/</\&lt\;/g;
966    $text =~ s/>/\&gt\;/g;
967    return $text;
968}
969
970
971sub unmake_text_xml_safe
972{
973    my $text = shift(@_);
974    $text =~ s/\&lt\;/</g;
975    $text =~ s/\&gt\;/>/g;
976    $text =~ s/\&amp\;/\&/g;
977    return $text;
978}
979
980
981# Returns 1 if $date1 is after $date2, 0 otherwise
982sub is_date_after_cvs
983{
984    my ($date1, $date2) = @_;
985    my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr",  4, "May",  5, "Jun",  6,
986          "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
987
988    if(!defined $date1) {
989        return 1;
990    }
991
992    my @date1parts = split(/-/, $date1);
993    my @date2parts = split(/-/, $date2);
994
995    # Compare year - nasty because we have rolled over into a new century
996    my $year1 = $date1parts[2];
997    if ($year1 < 80) {
998        $year1 += 2000;
999    }
1000    my $year2 = $date2parts[2];
1001    if ($year2 < 80) {
1002        $year2 += 2000;
1003    }
1004
1005    # Compare year
1006    if ($year1 > $year2) {
1007    return 1;
1008    }
1009    elsif ($year1 == $year2) {
1010    # Year is the same, so compare month
1011    if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1012        return 1;
1013    }
1014    elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1015        # Month is the same, so compare day
1016        if ($date1parts[0] > $date2parts[0]) {
1017        return 1;
1018        }
1019    }
1020    }
1021
1022    return 0;
1023}
1024
1025sub is_date_after
1026{
1027    my ($date1, $date2) = @_;
1028   
1029    if(!defined $date1) {
1030      return 1;
1031    }
1032    if(!defined $date2) {
1033      return 0;
1034    }
1035   
1036    # 16-Aug-2006
1037    if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1038       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1039          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1040       $date1=$3 . "-" . $months{$2} . "-" . $1;
1041       # print "** converted date1: $date1\n";
1042    }
1043    if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1044       my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr",  "04", "May",  "05", "Jun",  "06",
1045          "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1046       $date2=$3 . "-" . $months{$2} . "-" . $1;
1047       # print "** converted date2: $date2\n";
1048    }
1049   
1050   
1051    # 2006-08-16
1052    my @date1parts = split(/-/, $date1);
1053    my @date2parts = split(/-/, $date2);
1054   
1055    # Compare year
1056    if ($date1parts[0] > $date2parts[0]) {
1057    return 1;
1058    }
1059    elsif ($date1parts[0] == $date2parts[0]) {
1060    # Year is the same, so compare month
1061    if ($date1parts[1] > $date2parts[1]) {
1062        return 1;
1063    }
1064    elsif ($date1parts[1] == $date2parts[1]) {
1065        # Month is the same, so compare day
1066        if ($date1parts[2] > $date2parts[2]) {
1067        return 1;
1068        }
1069    }
1070    }   
1071   
1072    return 0;
1073}
1074
1075
1076sub create_xml_response_for_chunks_requiring_work
1077{
1078    my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1079
1080    # Form an XML response to the command
1081    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1082    $xml_response .= "<GTIResponse>\n";
1083    $xml_response .= "  <TranslationFile"
1084    . " key=\"" . $translation_file_key . "\""
1085    . " target_file_path=\"" . $target_file . "\""
1086    . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1087    . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1088    . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1089
1090    # Do chunks requiring translation first
1091    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1092    $xml_response .= "  <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1093    }
1094    else {
1095    $xml_response .= "  <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1096    }
1097
1098    my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1099    foreach my $chunk_key (@sorted_chunk_keys) {
1100    last if ($num_chunks_to_return == 0);
1101
1102    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1103    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key}); 
1104   
1105    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1106    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";   
1107    $xml_response .= "      <TargetFileText></TargetFileText>\n";
1108    $xml_response .= "    </Chunk>\n";
1109
1110    $num_chunks_to_return--;
1111    }
1112
1113    $xml_response .= "  </ChunksRequiringTranslation>\n";
1114
1115    # Then do chunks requiring updating
1116    if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1117    $xml_response .= "  <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1118    }
1119    else {
1120    $xml_response .= "  <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1121    }
1122       
1123    # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1124    @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1125    foreach my $chunk_key (@sorted_chunk_keys) {
1126    last if ($num_chunks_to_return == 0);
1127
1128    my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1129    my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1130    my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1131    my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1132
1133    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";   
1134    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1135    $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1136    $xml_response .= "    </Chunk>\n";
1137
1138    $num_chunks_to_return--;
1139    }
1140
1141    $xml_response .= "  </ChunksRequiringUpdating>\n";
1142
1143    $xml_response .= "</GTIResponse>\n";
1144
1145    return $xml_response;
1146}
1147
1148
1149sub create_xml_response_for_all_chunks
1150{
1151    my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1152
1153    # Form an XML response to the command
1154    my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1155    $xml_response .= "<GTIResponse>\n";
1156    $xml_response .= "  <TranslationFile"
1157    . " key=\"" . $translation_file_key . "\""
1158    . " target_file_path=\"" . $target_file . "\"\/>\n";
1159   
1160    # Do all the chunks
1161    $xml_response .= "  <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1162
1163    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1164    foreach my $chunk_key (@sorted_chunk_keys) {
1165    my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1166    my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1167
1168    $xml_response .= "    <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1169    $xml_response .= "      <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1170    if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1171        my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1172        my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1173        $xml_response .= "      <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1174    }
1175    else {
1176        $xml_response .= "      <TargetFileText></TargetFileText>\n";
1177    }
1178
1179    $xml_response .= "    </Chunk>\n";
1180    }
1181    $xml_response .= "  </Chunks>\n";
1182   
1183    $xml_response .= "</GTIResponse>\n";
1184    return $xml_response;
1185}
1186
1187
1188
1189# ==========================================================================================
1190#   MACROFILE FUNCTIONS
1191
1192sub build_key_to_line_mapping_for_macrofile
1193{
1194    my (@file_lines) = @_;
1195
1196    my $macro_package;
1197    my %chunk_key_to_line_mapping = ();
1198    # Process the contents of the file, line by line
1199    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1200    my $line = $file_lines[$i];
1201    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1202
1203    # Check if a new package is being defined
1204    if ($line =~ m/^package\s+(.+)/) {
1205        $macro_package = $1;
1206    }
1207
1208    # Line contains a macro name
1209    elsif ($line =~ m/^(_\w+_)/) {
1210        my $macro_key = $1;
1211        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1212
1213        # While there is still text of the macro to go...
1214        my $startline = $i;
1215        while ($line !~ /\}$/) {
1216        $i++;
1217        if ($i == scalar(@file_lines)) {
1218            &throw_fatal_error("Could not find end of macro $macro_key.");
1219        }
1220        $line = $file_lines[$i];
1221        $line =~ s/\s*([^\\]\#[^\}]+)?$//;  # Remove any comments and nasty whitespace
1222        }
1223
1224        # The chunk key consists of the package name and the macro key
1225        my $chunk_key = $macro_package . "." . $macro_key;
1226        # Map from chunk key to line
1227        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1228    }
1229
1230    # Icon: line in format ## "image text" ## image_type ## macro_name ##
1231    elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1232        # The chunk key consists of package name and macro key
1233        my $chunk_key = $macro_package . "." . $1;
1234        # Map from chunk key to line
1235        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1236    }
1237    }
1238
1239    return %chunk_key_to_line_mapping;
1240}
1241
1242
1243sub import_chunk_from_macrofile
1244{
1245    my ($chunk_text) = @_;
1246
1247    # Is this an icon macro??
1248    if ($chunk_text =~ /^\#\# (.*)/) {
1249    # Extract image macro text
1250    $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1251    $chunk_text = $1;
1252
1253    # Remove enclosing quotes
1254    $chunk_text =~ s/^\"//;
1255    $chunk_text =~ s/\"$//;
1256    }
1257
1258    # No, so it must be a text macro
1259    else {
1260    # Remove macro key
1261    $chunk_text =~ s/^_([^_]+)_(\s*)//;
1262
1263    # Remove language specifier
1264    $chunk_text =~ s/^\[l=.*\](\s*)//;
1265
1266    # Remove braces enclosing text
1267    $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1268    }
1269
1270    return $chunk_text;
1271}
1272
1273
1274sub get_macrofile_chunk_gti_comment
1275{
1276    my ($chunk_text) = @_;
1277
1278    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1279    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1280    return $1;
1281    }
1282
1283    return undef;
1284}
1285
1286
1287sub is_macrofile_chunk_automatically_translated
1288{
1289    my ($chunk_key) = @_;
1290
1291    # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1292    if ($chunk_key =~ /\._(httpicon|width|height)/) {
1293    return 1;
1294    }
1295
1296    return 0;
1297}
1298
1299
1300# Use the source file to generate a target file that is formatted the same
1301sub write_translated_macrofile
1302{
1303    my $source_file = shift(@_);  # Not used
1304    my @source_file_lines = @{shift(@_)};
1305    my $source_file_key_to_text_mapping = shift(@_);
1306    my $target_file = shift(@_);
1307    my @target_file_lines = @{shift(@_)};
1308    my $target_file_key_to_text_mapping = shift(@_);
1309    my $target_file_key_to_gti_comment_mapping = shift(@_);
1310    my $target_language_code = shift(@_);
1311
1312    # Build a mapping from source file line to chunk key
1313    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1314    my %source_file_line_to_key_mapping = ();
1315    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1316    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1317    }
1318    my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1319    my $source_file_line_number = 0;
1320
1321    # Build a mapping from target file line to chunk key
1322    my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1323    my %target_file_line_to_key_mapping = ();
1324    foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1325    $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1326    }
1327    my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1328
1329    # Write the new target file
1330    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1331    if (!open(TARGET_FILE, ">$target_file_path")) {
1332    &throw_fatal_error("Could not write target file $target_file_path.");
1333    }
1334
1335    # Use the header from the target file, to keep language and author information
1336    if (scalar(@target_file_line_keys) > 0) {
1337    my $target_file_line_number = 0;
1338    my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1339    while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1340        my $target_file_line = $target_file_lines[$target_file_line_number];
1341        last if ($target_file_line =~ /^\# -- Missing translation: /);  # We don't want to get into the macros
1342        print TARGET_FILE $target_file_line;
1343        $target_file_line_number++;
1344    }
1345
1346    $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1347    }
1348
1349    # Model the new target file on the source file, with the target file translations
1350    foreach my $line_key (@source_file_line_keys) {
1351    # Fill in the gaps before this chunk starts
1352    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1353    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1354    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1355        print TARGET_FILE $source_file_lines[$source_file_line_number];
1356        $source_file_line_number++;
1357    }
1358    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1359
1360    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1361    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1362    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1363
1364    my $macrofile_key = $chunk_key;
1365    $macrofile_key =~ s/^(.+?)\.//;
1366
1367    # If no translation exists for this chunk, show this, and move on
1368    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1369        print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1370        next;
1371    }
1372
1373    # Grab the source chunk text
1374    my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1375    for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1376        $source_file_chunk .= $source_file_lines[$l];
1377    }
1378
1379    # Is this an icon macro??
1380    if ($source_file_chunk =~ /^\#\# (.*)/) {
1381        # Escape any newline and question mark characters so the source text is replaced correctly
1382        $source_file_chunk_text =~ s/\\/\\\\/g;
1383        $source_file_chunk_text =~ s/\?/\\\?/g;
1384
1385        # Build the new target chunk from the source chunk
1386        my $target_file_chunk = $source_file_chunk;
1387        $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1388        $target_file_chunk =~ s/(\s)*$//;
1389        print TARGET_FILE "$target_file_chunk";
1390    }
1391
1392    # No, it is just a normal text macro
1393    else {
1394        print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1395    }
1396
1397    # Add the "updated" comment, if one exists
1398    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1399        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1400    }
1401    print TARGET_FILE "\n";
1402    }
1403
1404    close(TARGET_FILE);
1405}
1406
1407
1408sub sort_by_line
1409{
1410    return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1411}
1412
1413
1414# ==========================================================================================
1415#   RESOURCE BUNDLE FUNCTIONS
1416
1417sub build_key_to_line_mapping_for_resource_bundle
1418{
1419    my (@file_lines) = @_;
1420
1421    my %chunk_key_to_line_mapping = ();
1422    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1423    my $line = $file_lines[$i];
1424    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1425
1426    # Line contains a dictionary string
1427    if ($line =~ /^(\S+?)[:|=](.*)$/) {
1428        my $chunk_key = $1;
1429
1430        # Map from chunk key to line
1431        $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1432    }
1433    }
1434
1435    return %chunk_key_to_line_mapping;
1436}
1437
1438
1439sub import_chunk_from_resource_bundle
1440{
1441    my ($chunk_text) = @_;
1442
1443    # Simple: just remove string key
1444    $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1445    $chunk_text =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1446    $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1447
1448    return $chunk_text;
1449}
1450
1451
1452sub get_resource_bundle_chunk_gti_comment
1453{
1454    my ($chunk_text) = @_;
1455
1456    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1457    if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1458    return $1;
1459    }
1460
1461    return undef;
1462}
1463
1464
1465sub is_resource_bundle_chunk_automatically_translated
1466{
1467    # No resource bundle chunks are automatically translated
1468    return 0;
1469}
1470
1471
1472sub write_translated_resource_bundle
1473{
1474    my $source_file = shift(@_);  # Not used
1475    my @source_file_lines = @{shift(@_)};
1476    my $source_file_key_to_text_mapping = shift(@_);
1477    my $target_file = shift(@_);
1478    my @target_file_lines = @{shift(@_)};  # Not used
1479    my $target_file_key_to_text_mapping = shift(@_);
1480    my $target_file_key_to_gti_comment_mapping = shift(@_);
1481    my $target_language_code = shift(@_);  # Not used
1482
1483    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1484    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1485    my %source_file_line_to_key_mapping = ();
1486    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1487    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1488    }
1489
1490    # Write the new target file
1491    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1492    if (!open(TARGET_FILE, ">$target_file_path")) {
1493    &throw_fatal_error("Could not write target file $target_file_path.");
1494    }
1495
1496    # Model the new target file on the source file, with the target file translations
1497    my $source_file_line_number = 0;
1498    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1499    # Fill in the gaps before this chunk starts
1500    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1501    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1502    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1503        print TARGET_FILE $source_file_lines[$source_file_line_number];
1504        $source_file_line_number++;
1505    }
1506    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1507
1508    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1509    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1510    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1511
1512    # If no translation exists for this chunk, show this, and move on
1513    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1514        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1515        next;
1516    }
1517
1518    print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1519    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1520        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1521    }
1522    print TARGET_FILE "\n";
1523    }
1524
1525    close(TARGET_FILE);
1526}
1527
1528
1529# ==========================================================================================
1530#   GREENSTONE XML FUNCTIONS
1531
1532sub build_key_to_line_mapping_for_greenstone_xml
1533{
1534    my (@file_lines) = @_;
1535
1536    my %chunk_key_to_line_mapping = ();
1537    for (my $i = 0; $i < scalar(@file_lines); $i++) {
1538    my $line = $file_lines[$i];
1539    $line =~ s/(\s*)$//;  # Remove any nasty whitespace, carriage returns etc.
1540
1541    # Line contains a string to translate
1542    if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1543        my $chunk_key = $1;
1544        $line =~ s/\s*$//;  # Remove any nasty whitespace
1545        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1546
1547        # While there is still text of the string to go...
1548        my $startline = $i;
1549        while ($line !~ /<\/Text>$/) {
1550        $i++;
1551        if ($i == scalar(@file_lines)) {
1552            &throw_fatal_error("Could not find end of string $chunk_key.");
1553        }
1554        $line = $file_lines[$i];
1555        $line =~ s/\s*$//;  # Remove any nasty whitespace
1556        $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1557        }
1558
1559        # Map from chunk key to line
1560        if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1561        $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1562        }
1563        else {
1564        &throw_fatal_error("Duplicate key $chunk_key.");
1565        }
1566    }
1567    }
1568
1569    return %chunk_key_to_line_mapping;
1570}
1571
1572
1573sub import_chunk_from_greenstone_xml
1574{
1575    my ($chunk_text) = @_;
1576
1577    # Simple: just remove the Text tags
1578    $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1579    $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1580    $chunk_text =~ s/<\/Text>$//;
1581
1582    return $chunk_text;
1583}
1584
1585
1586sub get_greenstone_xml_chunk_gti_comment
1587{
1588    my ($chunk_text) = @_;
1589
1590    # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1591    if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1592    return $1;
1593    }
1594
1595    return undef;
1596}
1597
1598
1599sub is_greenstone_xml_chunk_automatically_translated
1600{
1601    # No greenstone XML chunks are automatically translated
1602    return 0;
1603}
1604
1605
1606sub write_translated_greenstone_xml
1607{
1608    my $source_file = shift(@_);  # Not used
1609    my @source_file_lines = @{shift(@_)};
1610    my $source_file_key_to_text_mapping = shift(@_);
1611    my $target_file = shift(@_);
1612    my @target_file_lines = @{shift(@_)};  # Not used
1613    my $target_file_key_to_text_mapping = shift(@_);
1614    my $target_file_key_to_gti_comment_mapping = shift(@_);
1615    my $target_language_code = shift(@_);  # Not used
1616
1617    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1618    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1619    my %source_file_line_to_key_mapping = ();
1620    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1621    $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1622    }
1623
1624    # Write the new target file
1625    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1626    if (!open(TARGET_FILE, ">$target_file_path")) {
1627    &throw_fatal_error("Could not write target file $target_file_path.");
1628    }
1629
1630    # Model the new target file on the source file, with the target file translations
1631    my $source_file_line_number = 0;
1632    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1633    # Fill in the gaps before this chunk starts
1634    my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1635    my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1636    while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1637        print TARGET_FILE $source_file_lines[$source_file_line_number];
1638        $source_file_line_number++;
1639    }
1640    $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1641
1642    my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1643    my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1644    my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1645    $target_file_chunk_text =~ s/(\n)*$//g;
1646
1647    # If no translation exists for this chunk, show this, and move on
1648    if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1649        print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1650        next;
1651    }
1652
1653    print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1654    if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1655        my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1656        $chunk_gti_comment =~ s/^Updated //;
1657        print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1658    }
1659    print TARGET_FILE "\n";
1660    }
1661
1662    # Fill in the end of the file
1663    while ($source_file_line_number < scalar(@source_file_lines)) {
1664    print TARGET_FILE $source_file_lines[$source_file_line_number];
1665    $source_file_line_number++;
1666    }
1667
1668    close(TARGET_FILE);
1669}
1670
1671
1672# ==========================================================================================
1673#   GREENSTONE3 FUNCTIONS
1674
1675sub get_all_chunks_for_gs3
1676{
1677    # The code of the target language (ensure it is lowercase)
1678    my $target_language_code = lc(shift(@_));
1679    my $translation_file_key = lc(shift(@_));
1680
1681    # Check that the necessary arguments were supplied
1682    if (!$target_language_code) {
1683    &throw_fatal_error("Missing command argument.");
1684    }
1685
1686    # Get (and check) the translation configuration
1687    # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1688   
1689    my %source_files_key_to_text_mapping = ();
1690    my %target_files_key_to_text_mapping = ();
1691    my %source_files_key_to_last_update_date_mapping = ();
1692    my %target_files_key_to_last_update_date_mapping = ();
1693
1694    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1695
1696    &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1697    &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1698
1699    my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);   
1700    return $xml_response;
1701}
1702
1703
1704sub get_first_n_chunks_requiring_work_for_gs3
1705{
1706    # The code of the target language (ensure it is lowercase)
1707    my $target_language_code = lc(shift(@_));
1708    # The key of the file to translate (ensure it is lowercase)
1709    my $translation_file_key = lc(shift(@_));
1710    # The number of chunks to return (defaults to one if not specified)
1711    my $num_chunks_to_return = shift(@_) || "1";
1712   
1713    # Check that the necessary arguments were supplied
1714    if (!$target_language_code || !$translation_file_key) {
1715    &throw_fatal_error("Missing command argument.");
1716    }
1717   
1718    my %source_files_key_to_text_mapping = ();
1719    my %target_files_key_to_text_mapping = ();
1720    my %source_files_key_to_last_update_date_mapping = ();
1721    my %target_files_key_to_last_update_date_mapping = ();
1722
1723    &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1724                 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1725   
1726    # Determine the target file chunks requiring translation
1727    my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);   
1728    # Determine the target file chunks requiring updating
1729    my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1730    &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1731    &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1732
1733    my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1734                                      \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1735                                      $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1736                                      \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1737   
1738    return $xml_response;
1739}
1740
1741
1742
1743sub build_gs3_configuration
1744{
1745    my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1746    $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1747   
1748    my $source_file_directory = "greenstone3";
1749    my $translation_file_type = "resource_bundle";
1750
1751    foreach my $interface_file_key (@gs3_interface_files) {
1752
1753    &log_message("Greenstone 3 interface file: " . $interface_file_key);
1754
1755    # Parse the source language and target language files
1756    my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1757    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1758    my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1759    my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1760    my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1761   
1762    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1763    my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1764    my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1765    my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1766    my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1767
1768
1769    # Filter out any automatically translated chunks
1770    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1771        if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1772        delete $source_file_key_to_line_mapping{$chunk_key};
1773        delete $target_file_key_to_line_mapping{$chunk_key};
1774        }
1775    }
1776
1777    &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1778    &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1779
1780    foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1781        my $global_chunk_key = "$interface_file_key.$chunk_key";
1782        $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1783        $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1784
1785        if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1786        $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1787        $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1788        }
1789    }   
1790    }
1791}
1792
1793
1794sub write_translated_gs3interface
1795{
1796    my $source_file_key_to_text_mapping = shift(@_);
1797    my $target_file_key_to_text_mapping = shift(@_);
1798    my $target_file_key_to_gti_comment_mapping = shift(@_);
1799    my $target_language_code = shift(@_);
1800   
1801    my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1802
1803    my %translated_interface_file_keys = ();
1804    foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1805    $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1806    if (!defined $translated_interface_file_keys{$1}) {
1807        &log_message("Updated interface file: " . $1); 
1808        $translated_interface_file_keys{$1}="";
1809    }
1810    }
1811    &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1812   
1813    my $source_file_directory = "greenstone3";   
1814   
1815    foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1816   
1817    # Build a mapping from chunk key to source file line, and from source file line to chunk key
1818    my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1819    my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1820    my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1821    my %source_file_line_to_key_mapping = ();
1822    foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1823        $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1824    }
1825   
1826    # Write the new target file
1827    my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1828    my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1829    if (!open(TARGET_FILE, ">$target_file_path")) {
1830        &throw_fatal_error("Could not write target file $target_file_path.");
1831    }
1832
1833    # Model the new target file on the source file, with the target file translations
1834    my $source_file_line_number = 0;
1835    foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1836        # Fill in the gaps before this chunk starts
1837        my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1838        my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1839        while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1840        print TARGET_FILE $source_file_lines[$source_file_line_number];
1841        $source_file_line_number++;
1842        }
1843        $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1844
1845        my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1846        my $global_chunk_key = "$interface_file_key.$chunk_key";
1847        my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1848        my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1849
1850        # If no translation exists for this chunk, show this, and move on
1851        if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1852        print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1853        next;
1854        }
1855
1856        print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1857        if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1858        print TARGET_FILE "  # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1859        }
1860        print TARGET_FILE "\n";
1861    }
1862
1863    close(TARGET_FILE);
1864    }           
1865}
1866
1867&main(@ARGV);
Note: See TracBrowser for help on using the browser.