source: main/trunk/greenstone2/bin/script/gti.pl@ 24362

Last change on this file since 24362 was 24362, checked in by ak19, 13 years ago

The method of locating perl has changed once more: util now defines the fuction get_perl_exec which is used by other scripts to obtain the path to the perl executable they should use.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 76.9 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35use iso639;
36use strict;
37use util;
38
39
40#my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
41#my $anonymous_svn_root = "http://http://svn.greenstone.org/gsdl/trunk/";
42my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
43my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
44my $source_language_code = "en"; # This is non-negiotable
45
46my $gti_translation_files =
47 [ # Greenstone macrofiles
48 { 'key' => "coredm",
49 'file_type' => "macrofile",
50 'source_file' => "macros/english.dm",
51 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
52 { 'key' => "auxdm",
53 'file_type' => "macrofile",
54 'source_file' => "macros/english2.dm",
55 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57 # GLI dictionary
58 { 'key' => "glidict",
59 'file_type' => "resource_bundle",
60 'source_file' => "gli/classes/dictionary.properties",
61 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
62
63 # GLI help
64 { 'key' => "glihelp",
65 'file_type' => "greenstone_xml",
66 'source_file' => "gli/help/en/help.xml",
67 'target_file' => "gli/help/{target_language_code}/help.xml" },
68
69 # Greenstone Perl modules
70 { 'key' => "perlmodules",
71 'file_type' => "resource_bundle",
72 'source_file' => "perllib/strings.properties",
73 'target_file' => "perllib/strings_{target_language_code}.properties" },
74
75 # Greenstone tutorial exercises
76 # { 'key' => "tutorials",
77 # 'file_type' => "greenstone_xml",
78 # 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
79 # 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
80
81 # new Greenstone.org
82 { 'key' => "greenorg",
83 'file_type' => "resource_bundle",
84 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
85 'target_file' => "greenstoneorg/website/classes/Gsc_{iso_639_1_target_language_name}.properties"
86 # 'file_type' => "macrofile",
87 # 'source_file' => "greenorg/macros/english.dm",
88 # 'target_file' => "greenorg/macros/{iso_639_1_target_language_name}.dm"
89 }
90
91 # { 'key' => "gs3interface",
92 #'file_type' => "resource_bundle",
93 #'source_file' => "greenstone3",
94 #'target_file' => "greenstone3"
95 #}
96 ];
97
98my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
99
100
101sub main
102{
103 # Get the command to process, and any arguments
104 my $gti_command = shift(@_);
105 my @gti_command_arguments = @_;
106 my $module = $_[1];
107
108 # Open the GTI log file for appending, or write to STDERR if that fails
109 if (!open(GTI_LOG, ">>$gti_log_file")) {
110 open(GTI_LOG, ">&STDERR");
111 }
112
113 # Log the command that launched this script
114 &log_message("Command: $0 @ARGV");
115
116 # Check that a command was supplied
117 if (!$gti_command) {
118 &throw_fatal_error("Missing command.");
119 }
120
121 # Process the command
122 if ($gti_command =~ /^get-all-chunks$/i) {
123 # Check that GS3 interface is the target
124 if ($module eq "gs3interface") {
125 print &get_all_chunks_for_gs3(@gti_command_arguments);
126 } else {
127 print &get_all_chunks(@gti_command_arguments);
128 }
129 }
130 if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
131 if ($module eq "gs3interface") {
132 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
133 } else {
134 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
135 }
136 }
137 if ($gti_command =~ /^get-language-status$/i) {
138 print &get_language_status(@gti_command_arguments);
139 }
140 if ($gti_command =~ /^search-chunks$/i) {
141 print &search_chunks(@gti_command_arguments);
142 }
143 if ($gti_command =~ /^submit-translations$/i) {
144 # This command cannot produce any output since it reads input
145 &submit_translations(@gti_command_arguments);
146 }
147 if ($gti_command =~ /^create-glihelp-zip-file$/i) {
148 # This command cannot produce any output since it reads input
149 &create_glihelp_zip_file(@gti_command_arguments);
150 }
151# else {
152 # The command was not recognized
153 #&throw_fatal_error("Unknown command \"$gti_command\".");
154 #}
155}
156
157
158sub throw_fatal_error
159{
160 my $error_message = shift(@_);
161
162 # Write an XML error response
163 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
164 print "<GTIResponse>\n";
165 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
166 print "</GTIResponse>\n";
167
168 # Log the error message, then die
169 &log_message("Error: $error_message");
170 die "\n";
171}
172
173
174sub log_message
175{
176 my $log_message = shift(@_);
177 print GTI_LOG time() . " -- " . $log_message . "\n";
178}
179
180
181sub get_all_chunks
182{
183 # The code of the target language (ensure it is lowercase)
184 my $target_language_code = lc(shift(@_));
185 # The key of the file to translate (ensure it is lowercase)
186 my $translation_file_key = lc(shift(@_));
187
188 # Check that the necessary arguments were supplied
189 if (!$target_language_code || !$translation_file_key) {
190 &throw_fatal_error("Missing command argument.");
191 }
192
193 # Get (and check) the translation configuration
194 my ($source_file, $target_file, $translation_file_type)
195 = &get_translation_configuration($target_language_code, $translation_file_key);
196
197 # Parse the source language and target language files
198 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
199 my @source_file_lines = &read_file_lines($source_file_path);
200 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
201
202 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
203 my @target_file_lines = &read_file_lines($target_file_path);
204 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
205
206 # Filter out any automatically translated chunks
207 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
208 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
209 delete $source_file_key_to_line_mapping{$chunk_key};
210 delete $target_file_key_to_line_mapping{$chunk_key};
211 }
212 }
213
214 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
215 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
216 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
217 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
218
219 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
220 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
221
222 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
223
224 return $xml_response;
225}
226
227
228sub get_first_n_chunks_requiring_work
229{
230 # The code of the target language (ensure it is lowercase)
231 my $target_language_code = lc(shift(@_));
232 # The key of the file to translate (ensure it is lowercase)
233 my $translation_file_key = lc(shift(@_));
234 # The number of chunks to return (defaults to one if not specified)
235 my $num_chunks_to_return = shift(@_) || "1";
236
237 # Check that the necessary arguments were supplied
238 if (!$target_language_code || !$translation_file_key) {
239 &throw_fatal_error("Missing command argument.");
240 }
241
242 # Get (and check) the translation configuration
243 my ($source_file, $target_file, $translation_file_type)
244 = &get_translation_configuration($target_language_code, $translation_file_key);
245
246 # Parse the source language and target language files
247 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
248 my @source_file_lines = &read_file_lines($source_file_path);
249 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
250
251 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
252 my @target_file_lines = &read_file_lines($target_file_path);
253 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
254
255 # Filter out any automatically translated chunks
256 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
257 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
258 delete $source_file_key_to_line_mapping{$chunk_key};
259 delete $target_file_key_to_line_mapping{$chunk_key};
260 }
261 }
262
263 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
264 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
265 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
266 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
267
268 # Determine the target file chunks requiring translation
269 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
270 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
271
272 # Determine the target file chunks requiring updating
273 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
274 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
275 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
276 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
277
278 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
279
280 return $xml_response;
281}
282
283
284sub get_language_status
285{
286 # The code of the target language (ensure it is lowercase)
287 my $target_language_code = lc(shift(@_));
288
289 # Check that the necessary arguments were supplied
290 if (!$target_language_code) {
291 &throw_fatal_error("Missing command argument.");
292 }
293
294 # Form an XML response to the command
295 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
296 $xml_response .= "<GTIResponse>\n";
297 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
298
299 foreach my $translation_file (@$gti_translation_files) {
300 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
301 my $target_file_name = "";
302
303 if ($translation_file->{'key'} eq "gs3interface") {
304 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
305 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
306
307 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
308 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
309
310 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
311 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
312 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
313 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
314 }
315 else {
316 # Get (and check) the translation configuration
317 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
318 $target_file_name = $target_file;
319
320 # Parse the source language and target language files
321 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
322 my @source_file_lines = &read_file_lines($source_file_path);
323 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
324
325 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
326 my @target_file_lines = &read_file_lines($target_file_path);
327 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
328
329 # Filter out any automatically translated chunks
330 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
331 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
332 delete $source_file_key_to_line_mapping{$chunk_key};
333 delete $target_file_key_to_line_mapping{$chunk_key};
334 }
335 }
336
337 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
338 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
339
340 # Determine the target file chunks requiring translation
341 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
342
343 # Determine the target file chunks requiring updating
344 my @target_file_keys_requiring_updating = ();
345 if (-e $target_file_path) {
346 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
347 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
348 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
349 }
350
351 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
352 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
353 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
354 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
355 }
356
357 &log_message("Status of " . $translation_file->{'key'});
358 &log_message("Number of source chunks: " . $num_source_chunks);
359 &log_message("Number of target chunks: " . $num_target_chunks);
360 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
361 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
362
363 $xml_response .= " <TranslationFile"
364 . " key=\"" . $translation_file->{'key'} . "\""
365 . " target_file_path=\"" . $target_file_name . "\""
366 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
367 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
368 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
369 }
370
371 $xml_response .= " </LanguageStatus>\n";
372
373 $xml_response .= "</GTIResponse>\n";
374 return $xml_response;
375}
376
377
378sub search_chunks
379{
380 # The code of the target language (ensure it is lowercase)
381 my $target_language_code = lc(shift(@_));
382 # The key of the file to translate (ensure it is lowercase)
383 my $translation_file_key = lc(shift(@_));
384 # The query string
385 my $query_string = join(' ', @_);
386
387 # Check that the necessary arguments were supplied
388 if (!$target_language_code || !$translation_file_key || !$query_string) {
389 &throw_fatal_error("Missing command argument.");
390 }
391
392 my ($source_file, $target_file, $translation_file_type) = ();
393 my %source_file_key_to_text_mapping = ();
394 my %target_file_key_to_text_mapping = ();
395
396
397 if ($translation_file_key ne "gs3interface") {
398 # Get (and check) the translation configuration
399 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
400
401 # Parse the source language and target language files
402 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
403 my @source_file_lines = &read_file_lines($source_file_path);
404 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
405
406 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
407 my @target_file_lines = &read_file_lines($target_file_path);
408 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
409
410 # Filter out any automatically translated chunks
411 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
412 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
413 delete $source_file_key_to_line_mapping{$chunk_key};
414 delete $target_file_key_to_line_mapping{$chunk_key};
415 }
416 }
417
418 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
419 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
420 }
421 else {
422 # Not needed in this case
423 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
424 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
425 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
426 }
427
428 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
429 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
430
431 # Determine the target file chunks matching the query
432 my @target_file_keys_matching_query = ();
433 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
434 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
435 if ($target_file_text =~ /$query_string/i) {
436 # &log_message("Chunk with key $chunk_key matches query.");
437 push(@target_file_keys_matching_query, $chunk_key);
438 }
439 }
440
441 # Form an XML response to the command
442 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
443 $xml_response .= "<GTIResponse>\n";
444
445 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
446 foreach my $chunk_key (@target_file_keys_matching_query) {
447 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
448
449 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
450 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
451 $xml_response .= " </Chunk>\n";
452 }
453 $xml_response .= " </ChunksMatchingQuery>\n";
454
455 $xml_response .= "</GTIResponse>\n";
456 return $xml_response;
457}
458
459
460sub submit_translations
461{
462 # The code of the target language (ensure it is lowercase)
463 my $target_language_code = lc(shift(@_));
464 # The key of the file to translate (ensure it is lowercase)
465 my $translation_file_key = lc(shift(@_));
466 # The username of the translation submitter
467 my $submitter_username = shift(@_);
468 # Whether to submit a target chunk even if it hasn't changed
469 my $force_submission_flag = shift(@_);
470
471 # Check that the necessary arguments were supplied
472 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
473 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
474 die "\n";
475 }
476
477 my %source_file_key_to_text_mapping = ();
478 my %source_file_key_to_gti_comment_mapping = ();
479 my %target_file_key_to_text_mapping = ();
480 my %target_file_key_to_gti_comment_mapping = ();
481
482 my (@source_file_lines, @target_file_lines) = ();
483 my ($source_file, $target_file, $translation_file_type);
484
485
486 if ($translation_file_key ne "gs3interface") {
487 # Get (and check) the translation configuration
488 ($source_file, $target_file, $translation_file_type)
489 = &get_translation_configuration($target_language_code, $translation_file_key);
490
491 # Parse the source language and target language files
492 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
493 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
494 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
495 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
496
497 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
498 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
499 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
500 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
501 }
502 else {
503 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
504 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
505 }
506 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
507 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
508
509 # Submission date
510 my $day = (localtime)[3];
511 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
512 my $year = (localtime)[5] + 1900;
513 my $submission_date = "$day-$month-$year";
514
515 open(SUBMISSION, "-");
516 my @submission_lines = <SUBMISSION>;
517 close(SUBMISSION);
518
519 # Remove any nasty carriage returns
520 # &log_message("Submission:");
521 foreach my $submission_line (@submission_lines) {
522 $submission_line =~ s/\r$//;
523 #&log_message(" $submission_line");
524 }
525
526 my %source_file_key_to_submission_mapping = ();
527 my %target_file_key_to_submission_mapping = ();
528 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
529 # Read source file part of submission
530 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
531 my $chunk_key = $1;
532
533 # Read the source file text
534 my $source_file_chunk_text = "";
535 $i++;
536 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
537 $source_file_chunk_text .= $submission_lines[$i];
538 $i++;
539 }
540 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
541 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
542
543 #&log_message("Source file key: $chunk_key");
544 #&log_message("Source file text: $source_file_chunk_text");
545 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
546 }
547
548 # Read target file part of submission
549 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
550 my $chunk_key = $1;
551
552 # Read the target file text
553 my $target_file_chunk_text = "";
554 $i++;
555 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
556 $target_file_chunk_text .= $submission_lines[$i];
557 $i++;
558 }
559 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
560 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
561
562 #&log_message("Target file key: $chunk_key");
563 #&log_message("Target file text: $target_file_chunk_text");
564 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
565 }
566 }
567
568 # -----------------------------------------
569 # Validate the translation submissions
570 # -----------------------------------------
571
572 # Check that the translations are valid
573 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
574 # Make sure the submitted chunk still exists in the source file
575 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
576 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
577 delete $source_file_key_to_submission_mapping{$chunk_key};
578 delete $target_file_key_to_submission_mapping{$chunk_key};
579 next;
580 }
581
582 # Make sure the submitted source chunk matches the source file chunk
583 if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
584 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
585 &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
586 &log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
587 delete $source_file_key_to_submission_mapping{$chunk_key};
588 delete $target_file_key_to_submission_mapping{$chunk_key};
589 next;
590 }
591 }
592
593 # Apply the submitted translations
594 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
595 # Only apply the submission if it is a change, unless -force_submission has been specified
596 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
597 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
598 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
599 }
600 }
601
602 if ($translation_file_key ne "gs3interface") {
603 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
604 } else {
605 eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
606 }
607}
608
609
610sub create_glihelp_zip_file
611{
612 my $target_language_code = shift(@_);
613 my $translation_file_key = "glihelp";
614
615 &log_message("Creating GLI Help zip file for $target_language_code");
616
617 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
618
619 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
620 if ( ! -e $classpath) {
621 &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
622 }
623
624 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
625 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
626
627 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
628 if ( ! -e $gen_many_html_xsl_filepath) {
629 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
630 }
631
632 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
633 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
634
635 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
636 $target_file_directory = $target_file_directory."/";
637
638 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
639 my $perl_exec = &util::get_perl_exec();
640 my $java_exec = "java";
641 if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
642 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
643 }
644
645 my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
646 my $response = `$cmd`;
647 &log_message($cmd);
648 &log_message($response);
649 &log_message("Created HTML operational files");
650
651 $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
652 $response = `$cmd`;
653 &log_message($cmd);
654 &log_message($response);
655 &log_message("Created xml index file");
656
657 my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
658 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
659 $response = `$cmd`;
660 &log_message($cmd);
661 &log_message($response);
662 &log_message("Created the zip file");
663}
664
665
666sub get_translation_configuration
667{
668 # Get the code of the target language
669 my $target_language_code = shift(@_);
670 # Get the key of the file to translate
671 my $translation_file_key = shift(@_);
672
673 # Read the translation data from the gti.cfg file
674 my ($source_file, $target_file, $translation_file_type) =
675 &get_translation_data_for($target_language_code, $translation_file_key);
676
677 # Check that the file to translate is defined in the gti.cfg file
678 if (!$source_file || !$target_file || !$translation_file_type) {
679 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
680 }
681
682 # Check that the source file exists
683 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
684 if (!-e $source_file_path) {
685 &throw_fatal_error("Source file $source_file_path does not exist.");
686 }
687
688 # Check that the source file is up to date
689 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
690 # unless ($translation_file_is_not_in_cvs) {
691 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
692 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
693 if ($source_file_cvs_status =~ /^C /) {
694 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
695 }
696 if ($source_file_cvs_status =~ /^M /) {
697 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
698 }
699 # }
700
701 return ($source_file, $target_file, $translation_file_type);
702}
703
704
705sub get_translation_data_for
706{
707 my ($target_language_code, $translation_file_key) = @_;
708
709 foreach my $translation_file (@$gti_translation_files) {
710 # If this isn't the correct translation file, move onto the next one
711 next if ($translation_file_key ne $translation_file->{'key'});
712
713 # Resolve the target language file
714 my $target_language_file = $translation_file->{'target_file'};
715 if ($target_language_file =~ /(\{.+\;.+\})/) {
716 my $unresolved_target_language_file_part = $1;
717
718 # Check for a special case for the target language code
719 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
720 my $resolved_target_language_file_part = $2;
721 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
722 }
723 # Otherwise use the last part as the default value
724 else {
725 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
726 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
727 }
728 }
729
730 # Resolve instances of {iso_639_1_target_language_name}
731 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
732 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
733 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
734
735 # Resolve instances of {target_language_code}
736 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
737
738 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
739 }
740
741 return ();
742}
743
744
745sub read_file_lines
746{
747 my ($file_path) = @_;
748
749 if (!open(FILE_IN, "<$file_path")) {
750 &log_message("Note: Could not open file $file_path.");
751 return ();
752 }
753 my @file_lines = <FILE_IN>;
754 close(FILE_IN);
755
756 return @file_lines;
757}
758
759
760sub build_key_to_line_mapping
761{
762 my ($file_lines, $translation_file_type) = @_;
763 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
764}
765
766
767sub build_key_to_text_mapping
768{
769 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
770
771 my %key_to_text_mapping = ();
772 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
773 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
774 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
775
776 my $chunk_text = @$file_lines[$chunk_starting_line];
777 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
778 $chunk_text .= @$file_lines[$l];
779 }
780
781 # Map from chunk key to text
782 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
783 }
784
785 return %key_to_text_mapping;
786}
787
788
789sub build_key_to_last_update_date_mapping
790{
791 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
792
793 # If the files aren't in CVS then we can't tell anything about what needs updating
794 # return () if ($translation_file_is_not_in_cvs);
795
796 # Build a mapping from key to CVS date
797 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
798 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
799
800 # Build a mapping from key to comment date
801 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
802
803 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
804 my %key_to_last_update_date_mapping = ();
805 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
806 # Use the CVS date as a starting point
807 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
808 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
809
810 # If a comment date exists and it is after the CVS date, use that instead
811 # need to convert the comment date format to SVN format
812 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
813 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
814 my $chunk_comment_date = $1;
815 # if the changes were made on 2009 Jan 27, ie. tidying up the GTI commands following each fragment,
816 # then should use the GTI comment date instead
817 # if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
818 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date)
819 || (&is_date_after($chunk_cvs_date,$chunk_comment_date) && ($chunk_cvs_date eq "2009-01-27" || $chunk_cvs_date eq "2009-01-28")))) {
820 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
821
822 # &log_message("cvs date: $chunk_cvs_date, comment date: $chunk_comment_date");
823 # die;
824 }
825 }
826 }
827
828 return %key_to_last_update_date_mapping;
829}
830
831
832sub build_key_to_cvs_date_mapping
833{
834 my ($filename, $translation_file_type) = @_;
835
836 # Use CVS to annotate each line of the file with the date it was last edited
837 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
838 # my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;
839 # my $cvs_annotated_file = `cd $gsdl_root_directory; export PATH=.:/research/lh92/programs/subversion/bin; svn annotate -v --force $filename`;
840 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v $filename`;
841
842 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
843
844 my @cvs_annotated_file_lines_date = ();
845 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
846 # Extract the date from the CVS annotation at the front
847 # cvs format : 07-Jun-02
848 # svn format : 2007-07-16
849 # $cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
850 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
851
852 push(@cvs_annotated_file_lines_date, $1);
853
854 # trim extra date information in svn annotation format
855 # 15:42:49 +1200 (Wed, 21 Jun 2006)
856 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
857 }
858
859 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
860 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
861
862 my %key_to_cvs_date_mapping = ();
863 foreach my $chunk_key (keys(%key_to_line_mapping)) {
864 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
865 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
866
867 # Find the date this chunk was last edited, from the CVS annotation
868 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
869 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
870 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
871 # This part of the chunk has been updated more recently
872 $chunk_date = $cvs_annotated_file_lines_date[$l];
873
874 }
875 }
876
877 # Map from chunk key to CVS date
878 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
879 }
880
881 return %key_to_cvs_date_mapping;
882}
883
884
885sub build_key_to_gti_comment_mapping
886{
887 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
888
889 my %key_to_gti_comment_mapping = ();
890 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
891 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
892 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
893
894 my $chunk_text = @$file_lines[$chunk_starting_line];
895 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
896 $chunk_text .= @$file_lines[$l];
897 }
898
899 # Map from chunk key to GTI comment
900 my $chunk_gti_comment;
901 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
902 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
903 }
904
905 return %key_to_gti_comment_mapping;
906}
907
908
909sub determine_chunks_requiring_translation
910{
911 my $source_file_key_to_text_mapping = shift(@_);
912 my $target_file_key_to_text_mapping = shift(@_);
913
914 # Chunks needing translation are those in the source file with no translation in the target file
915 my @target_file_keys_requiring_translation = ();
916 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
917 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
918 # &log_message("Chunk with key $chunk_key needs translating.");
919 push(@target_file_keys_requiring_translation, $chunk_key);
920 }
921 }
922
923 return @target_file_keys_requiring_translation;
924}
925
926
927sub determine_chunks_requiring_updating
928{
929 my $source_file_key_to_last_update_date_mapping = shift(@_);
930 my $target_file_key_to_last_update_date_mapping = shift(@_);
931
932 # Chunks needing updating are those in the target file that have been more recently edited in the source file
933 my @target_file_keys_requiring_updating = ();
934 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
935 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
936 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
937
938 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
939
940 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
941 # &log_message("Chunk with key $chunk_key needs updating.");
942 push(@target_file_keys_requiring_updating, $chunk_key);
943 }
944 }
945
946 return @target_file_keys_requiring_updating;
947}
948
949
950sub is_chunk_automatically_translated
951{
952 my ($chunk_key, $translation_file_type) = @_;
953 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
954}
955
956
957sub make_text_xml_safe
958{
959 my $text = shift(@_);
960 $text =~ s/\&/\&amp\;/g;
961 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
962 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
963 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
964 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
965 $text =~ s/</\&lt\;/g;
966 $text =~ s/>/\&gt\;/g;
967 return $text;
968}
969
970
971sub unmake_text_xml_safe
972{
973 my $text = shift(@_);
974 $text =~ s/\&lt\;/</g;
975 $text =~ s/\&gt\;/>/g;
976 $text =~ s/\&amp\;/\&/g;
977 return $text;
978}
979
980
981# Returns 1 if $date1 is after $date2, 0 otherwise
982sub is_date_after_cvs
983{
984 my ($date1, $date2) = @_;
985 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
986 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
987
988 if(!defined $date1) {
989 return 1;
990 }
991
992 my @date1parts = split(/-/, $date1);
993 my @date2parts = split(/-/, $date2);
994
995 # Compare year - nasty because we have rolled over into a new century
996 my $year1 = $date1parts[2];
997 if ($year1 < 80) {
998 $year1 += 2000;
999 }
1000 my $year2 = $date2parts[2];
1001 if ($year2 < 80) {
1002 $year2 += 2000;
1003 }
1004
1005 # Compare year
1006 if ($year1 > $year2) {
1007 return 1;
1008 }
1009 elsif ($year1 == $year2) {
1010 # Year is the same, so compare month
1011 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1012 return 1;
1013 }
1014 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1015 # Month is the same, so compare day
1016 if ($date1parts[0] > $date2parts[0]) {
1017 return 1;
1018 }
1019 }
1020 }
1021
1022 return 0;
1023}
1024
1025sub is_date_after
1026{
1027 my ($date1, $date2) = @_;
1028
1029 if(!defined $date1) {
1030 return 1;
1031 }
1032 if(!defined $date2) {
1033 return 0;
1034 }
1035
1036 # 16-Aug-2006
1037 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1038 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1039 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1040 $date1=$3 . "-" . $months{$2} . "-" . $1;
1041 # print "** converted date1: $date1\n";
1042 }
1043 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1044 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1045 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1046 $date2=$3 . "-" . $months{$2} . "-" . $1;
1047 # print "** converted date2: $date2\n";
1048 }
1049
1050
1051 # 2006-08-16
1052 my @date1parts = split(/-/, $date1);
1053 my @date2parts = split(/-/, $date2);
1054
1055 # Compare year
1056 if ($date1parts[0] > $date2parts[0]) {
1057 return 1;
1058 }
1059 elsif ($date1parts[0] == $date2parts[0]) {
1060 # Year is the same, so compare month
1061 if ($date1parts[1] > $date2parts[1]) {
1062 return 1;
1063 }
1064 elsif ($date1parts[1] == $date2parts[1]) {
1065 # Month is the same, so compare day
1066 if ($date1parts[2] > $date2parts[2]) {
1067 return 1;
1068 }
1069 }
1070 }
1071
1072 return 0;
1073}
1074
1075
1076sub create_xml_response_for_chunks_requiring_work
1077{
1078 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1079
1080 # Form an XML response to the command
1081 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1082 $xml_response .= "<GTIResponse>\n";
1083 $xml_response .= " <TranslationFile"
1084 . " key=\"" . $translation_file_key . "\""
1085 . " target_file_path=\"" . $target_file . "\""
1086 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1087 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1088 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1089
1090 # Do chunks requiring translation first
1091 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1092 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1093 }
1094 else {
1095 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1096 }
1097
1098 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1099 foreach my $chunk_key (@sorted_chunk_keys) {
1100 last if ($num_chunks_to_return == 0);
1101
1102 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1103 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1104
1105 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1106 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1107 $xml_response .= " <TargetFileText></TargetFileText>\n";
1108 $xml_response .= " </Chunk>\n";
1109
1110 $num_chunks_to_return--;
1111 }
1112
1113 $xml_response .= " </ChunksRequiringTranslation>\n";
1114
1115 # Then do chunks requiring updating
1116 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1117 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1118 }
1119 else {
1120 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1121 }
1122
1123 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1124 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1125 foreach my $chunk_key (@sorted_chunk_keys) {
1126 last if ($num_chunks_to_return == 0);
1127
1128 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1129 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1130 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1131 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1132
1133 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1134 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1135 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1136 $xml_response .= " </Chunk>\n";
1137
1138 $num_chunks_to_return--;
1139 }
1140
1141 $xml_response .= " </ChunksRequiringUpdating>\n";
1142
1143 $xml_response .= "</GTIResponse>\n";
1144
1145 return $xml_response;
1146}
1147
1148
1149sub create_xml_response_for_all_chunks
1150{
1151 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1152
1153 # Form an XML response to the command
1154 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1155 $xml_response .= "<GTIResponse>\n";
1156 $xml_response .= " <TranslationFile"
1157 . " key=\"" . $translation_file_key . "\""
1158 . " target_file_path=\"" . $target_file . "\"\/>\n";
1159
1160 # Do all the chunks
1161 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1162
1163 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1164 foreach my $chunk_key (@sorted_chunk_keys) {
1165 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1166 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1167
1168 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1169 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1170 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1171 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1172 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1173 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1174 }
1175 else {
1176 $xml_response .= " <TargetFileText></TargetFileText>\n";
1177 }
1178
1179 $xml_response .= " </Chunk>\n";
1180 }
1181 $xml_response .= " </Chunks>\n";
1182
1183 $xml_response .= "</GTIResponse>\n";
1184 return $xml_response;
1185}
1186
1187
1188
1189# ==========================================================================================
1190# MACROFILE FUNCTIONS
1191
1192sub build_key_to_line_mapping_for_macrofile
1193{
1194 my (@file_lines) = @_;
1195
1196 my $macro_package;
1197 my %chunk_key_to_line_mapping = ();
1198 # Process the contents of the file, line by line
1199 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1200 my $line = $file_lines[$i];
1201 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1202
1203 # Check if a new package is being defined
1204 if ($line =~ m/^package\s+(.+)/) {
1205 $macro_package = $1;
1206 }
1207
1208 # Line contains a macro name
1209 elsif ($line =~ m/^(_\w+_)/) {
1210 my $macro_key = $1;
1211 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1212
1213 # While there is still text of the macro to go...
1214 my $startline = $i;
1215 while ($line !~ /\}$/) {
1216 $i++;
1217 if ($i == scalar(@file_lines)) {
1218 &throw_fatal_error("Could not find end of macro $macro_key.");
1219 }
1220 $line = $file_lines[$i];
1221 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1222 }
1223
1224 # The chunk key consists of the package name and the macro key
1225 my $chunk_key = $macro_package . "." . $macro_key;
1226 # Map from chunk key to line
1227 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1228 }
1229
1230 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1231 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1232 # The chunk key consists of package name and macro key
1233 my $chunk_key = $macro_package . "." . $1;
1234 # Map from chunk key to line
1235 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1236 }
1237 }
1238
1239 return %chunk_key_to_line_mapping;
1240}
1241
1242
1243sub import_chunk_from_macrofile
1244{
1245 my ($chunk_text) = @_;
1246
1247 # Is this an icon macro??
1248 if ($chunk_text =~ /^\#\# (.*)/) {
1249 # Extract image macro text
1250 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1251 $chunk_text = $1;
1252
1253 # Remove enclosing quotes
1254 $chunk_text =~ s/^\"//;
1255 $chunk_text =~ s/\"$//;
1256 }
1257
1258 # No, so it must be a text macro
1259 else {
1260 # Remove macro key
1261 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1262
1263 # Remove language specifier
1264 $chunk_text =~ s/^\[l=.*\](\s*)//;
1265
1266 # Remove braces enclosing text
1267 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1268 }
1269
1270 return $chunk_text;
1271}
1272
1273
1274sub get_macrofile_chunk_gti_comment
1275{
1276 my ($chunk_text) = @_;
1277
1278 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1279 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1280 return $1;
1281 }
1282
1283 return undef;
1284}
1285
1286
1287sub is_macrofile_chunk_automatically_translated
1288{
1289 my ($chunk_key) = @_;
1290
1291 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1292 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1293 return 1;
1294 }
1295
1296 return 0;
1297}
1298
1299
1300# Use the source file to generate a target file that is formatted the same
1301sub write_translated_macrofile
1302{
1303 my $source_file = shift(@_); # Not used
1304 my @source_file_lines = @{shift(@_)};
1305 my $source_file_key_to_text_mapping = shift(@_);
1306 my $target_file = shift(@_);
1307 my @target_file_lines = @{shift(@_)};
1308 my $target_file_key_to_text_mapping = shift(@_);
1309 my $target_file_key_to_gti_comment_mapping = shift(@_);
1310 my $target_language_code = shift(@_);
1311
1312 # Build a mapping from source file line to chunk key
1313 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1314 my %source_file_line_to_key_mapping = ();
1315 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1316 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1317 }
1318 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1319 my $source_file_line_number = 0;
1320
1321 # Build a mapping from target file line to chunk key
1322 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1323 my %target_file_line_to_key_mapping = ();
1324 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1325 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1326 }
1327 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1328
1329 # Write the new target file
1330 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1331 if (!open(TARGET_FILE, ">$target_file_path")) {
1332 &throw_fatal_error("Could not write target file $target_file_path.");
1333 }
1334
1335 # Use the header from the target file, to keep language and author information
1336 if (scalar(@target_file_line_keys) > 0) {
1337 my $target_file_line_number = 0;
1338 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1339 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1340 my $target_file_line = $target_file_lines[$target_file_line_number];
1341 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1342 print TARGET_FILE $target_file_line;
1343 $target_file_line_number++;
1344 }
1345
1346 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1347 }
1348
1349 # Model the new target file on the source file, with the target file translations
1350 foreach my $line_key (@source_file_line_keys) {
1351 # Fill in the gaps before this chunk starts
1352 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1353 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1354 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1355 print TARGET_FILE $source_file_lines[$source_file_line_number];
1356 $source_file_line_number++;
1357 }
1358 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1359
1360 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1361 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1362 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1363
1364 my $macrofile_key = $chunk_key;
1365 $macrofile_key =~ s/^(.+?)\.//;
1366
1367 # If no translation exists for this chunk, show this, and move on
1368 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1369 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1370 next;
1371 }
1372
1373 # Grab the source chunk text
1374 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1375 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1376 $source_file_chunk .= $source_file_lines[$l];
1377 }
1378
1379 # Is this an icon macro??
1380 if ($source_file_chunk =~ /^\#\# (.*)/) {
1381 # Escape any newline and question mark characters so the source text is replaced correctly
1382 $source_file_chunk_text =~ s/\\/\\\\/g;
1383 $source_file_chunk_text =~ s/\?/\\\?/g;
1384
1385 # Build the new target chunk from the source chunk
1386 my $target_file_chunk = $source_file_chunk;
1387 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1388 $target_file_chunk =~ s/(\s)*$//;
1389 print TARGET_FILE "$target_file_chunk";
1390 }
1391
1392 # No, it is just a normal text macro
1393 else {
1394 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1395 }
1396
1397 # Add the "updated" comment, if one exists
1398 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1399 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1400 }
1401 print TARGET_FILE "\n";
1402 }
1403
1404 close(TARGET_FILE);
1405}
1406
1407
1408sub sort_by_line
1409{
1410 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1411}
1412
1413
1414# ==========================================================================================
1415# RESOURCE BUNDLE FUNCTIONS
1416
1417sub build_key_to_line_mapping_for_resource_bundle
1418{
1419 my (@file_lines) = @_;
1420
1421 my %chunk_key_to_line_mapping = ();
1422 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1423 my $line = $file_lines[$i];
1424 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1425
1426 # Line contains a dictionary string
1427 if ($line =~ /^(\S+?)[:|=](.*)$/) {
1428 my $chunk_key = $1;
1429
1430 # Map from chunk key to line
1431 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1432 }
1433 }
1434
1435 return %chunk_key_to_line_mapping;
1436}
1437
1438
1439sub import_chunk_from_resource_bundle
1440{
1441 my ($chunk_text) = @_;
1442
1443 # Simple: just remove string key
1444 $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1445 $chunk_text =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1446 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1447
1448 return $chunk_text;
1449}
1450
1451
1452sub get_resource_bundle_chunk_gti_comment
1453{
1454 my ($chunk_text) = @_;
1455
1456 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1457 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1458 return $1;
1459 }
1460
1461 return undef;
1462}
1463
1464
1465sub is_resource_bundle_chunk_automatically_translated
1466{
1467 # No resource bundle chunks are automatically translated
1468 return 0;
1469}
1470
1471
1472sub write_translated_resource_bundle
1473{
1474 my $source_file = shift(@_); # Not used
1475 my @source_file_lines = @{shift(@_)};
1476 my $source_file_key_to_text_mapping = shift(@_);
1477 my $target_file = shift(@_);
1478 my @target_file_lines = @{shift(@_)}; # Not used
1479 my $target_file_key_to_text_mapping = shift(@_);
1480 my $target_file_key_to_gti_comment_mapping = shift(@_);
1481 my $target_language_code = shift(@_); # Not used
1482
1483 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1484 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1485 my %source_file_line_to_key_mapping = ();
1486 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1487 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1488 }
1489
1490 # Write the new target file
1491 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1492 if (!open(TARGET_FILE, ">$target_file_path")) {
1493 &throw_fatal_error("Could not write target file $target_file_path.");
1494 }
1495
1496 # Model the new target file on the source file, with the target file translations
1497 my $source_file_line_number = 0;
1498 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1499 # Fill in the gaps before this chunk starts
1500 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1501 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1502 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1503 print TARGET_FILE $source_file_lines[$source_file_line_number];
1504 $source_file_line_number++;
1505 }
1506 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1507
1508 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1509 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1510 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1511
1512 # If no translation exists for this chunk, show this, and move on
1513 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1514 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1515 next;
1516 }
1517
1518 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1519 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1520 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1521 }
1522 print TARGET_FILE "\n";
1523 }
1524
1525 close(TARGET_FILE);
1526}
1527
1528
1529# ==========================================================================================
1530# GREENSTONE XML FUNCTIONS
1531
1532sub build_key_to_line_mapping_for_greenstone_xml
1533{
1534 my (@file_lines) = @_;
1535
1536 my %chunk_key_to_line_mapping = ();
1537 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1538 my $line = $file_lines[$i];
1539 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1540
1541 # Line contains a string to translate
1542 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1543 my $chunk_key = $1;
1544 $line =~ s/\s*$//; # Remove any nasty whitespace
1545 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1546
1547 # While there is still text of the string to go...
1548 my $startline = $i;
1549 while ($line !~ /<\/Text>$/) {
1550 $i++;
1551 if ($i == scalar(@file_lines)) {
1552 &throw_fatal_error("Could not find end of string $chunk_key.");
1553 }
1554 $line = $file_lines[$i];
1555 $line =~ s/\s*$//; # Remove any nasty whitespace
1556 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1557 }
1558
1559 # Map from chunk key to line
1560 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1561 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1562 }
1563 else {
1564 &throw_fatal_error("Duplicate key $chunk_key.");
1565 }
1566 }
1567 }
1568
1569 return %chunk_key_to_line_mapping;
1570}
1571
1572
1573sub import_chunk_from_greenstone_xml
1574{
1575 my ($chunk_text) = @_;
1576
1577 # Simple: just remove the Text tags
1578 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1579 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1580 $chunk_text =~ s/<\/Text>$//;
1581
1582 return $chunk_text;
1583}
1584
1585
1586sub get_greenstone_xml_chunk_gti_comment
1587{
1588 my ($chunk_text) = @_;
1589
1590 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1591 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1592 return $1;
1593 }
1594
1595 return undef;
1596}
1597
1598
1599sub is_greenstone_xml_chunk_automatically_translated
1600{
1601 # No greenstone XML chunks are automatically translated
1602 return 0;
1603}
1604
1605
1606sub write_translated_greenstone_xml
1607{
1608 my $source_file = shift(@_); # Not used
1609 my @source_file_lines = @{shift(@_)};
1610 my $source_file_key_to_text_mapping = shift(@_);
1611 my $target_file = shift(@_);
1612 my @target_file_lines = @{shift(@_)}; # Not used
1613 my $target_file_key_to_text_mapping = shift(@_);
1614 my $target_file_key_to_gti_comment_mapping = shift(@_);
1615 my $target_language_code = shift(@_); # Not used
1616
1617 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1618 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1619 my %source_file_line_to_key_mapping = ();
1620 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1621 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1622 }
1623
1624 # Write the new target file
1625 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1626 if (!open(TARGET_FILE, ">$target_file_path")) {
1627 &throw_fatal_error("Could not write target file $target_file_path.");
1628 }
1629
1630 # Model the new target file on the source file, with the target file translations
1631 my $source_file_line_number = 0;
1632 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1633 # Fill in the gaps before this chunk starts
1634 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1635 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1636 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1637 print TARGET_FILE $source_file_lines[$source_file_line_number];
1638 $source_file_line_number++;
1639 }
1640 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1641
1642 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1643 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1644 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1645 $target_file_chunk_text =~ s/(\n)*$//g;
1646
1647 # If no translation exists for this chunk, show this, and move on
1648 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1649 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1650 next;
1651 }
1652
1653 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1654 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1655 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1656 $chunk_gti_comment =~ s/^Updated //;
1657 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1658 }
1659 print TARGET_FILE "\n";
1660 }
1661
1662 # Fill in the end of the file
1663 while ($source_file_line_number < scalar(@source_file_lines)) {
1664 print TARGET_FILE $source_file_lines[$source_file_line_number];
1665 $source_file_line_number++;
1666 }
1667
1668 close(TARGET_FILE);
1669}
1670
1671
1672# ==========================================================================================
1673# GREENSTONE3 FUNCTIONS
1674
1675sub get_all_chunks_for_gs3
1676{
1677 # The code of the target language (ensure it is lowercase)
1678 my $target_language_code = lc(shift(@_));
1679 my $translation_file_key = lc(shift(@_));
1680
1681 # Check that the necessary arguments were supplied
1682 if (!$target_language_code) {
1683 &throw_fatal_error("Missing command argument.");
1684 }
1685
1686 # Get (and check) the translation configuration
1687 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1688
1689 my %source_files_key_to_text_mapping = ();
1690 my %target_files_key_to_text_mapping = ();
1691 my %source_files_key_to_last_update_date_mapping = ();
1692 my %target_files_key_to_last_update_date_mapping = ();
1693
1694 &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1695
1696 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1697 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1698
1699 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1700 return $xml_response;
1701}
1702
1703
1704sub get_first_n_chunks_requiring_work_for_gs3
1705{
1706 # The code of the target language (ensure it is lowercase)
1707 my $target_language_code = lc(shift(@_));
1708 # The key of the file to translate (ensure it is lowercase)
1709 my $translation_file_key = lc(shift(@_));
1710 # The number of chunks to return (defaults to one if not specified)
1711 my $num_chunks_to_return = shift(@_) || "1";
1712
1713 # Check that the necessary arguments were supplied
1714 if (!$target_language_code || !$translation_file_key) {
1715 &throw_fatal_error("Missing command argument.");
1716 }
1717
1718 my %source_files_key_to_text_mapping = ();
1719 my %target_files_key_to_text_mapping = ();
1720 my %source_files_key_to_last_update_date_mapping = ();
1721 my %target_files_key_to_last_update_date_mapping = ();
1722
1723 &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1724 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1725
1726 # Determine the target file chunks requiring translation
1727 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1728 # Determine the target file chunks requiring updating
1729 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1730 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1731 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1732
1733 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1734 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1735 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1736 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1737
1738 return $xml_response;
1739}
1740
1741
1742
1743sub build_gs3_configuration
1744{
1745 my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1746 $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1747
1748 my $source_file_directory = "greenstone3";
1749 my $translation_file_type = "resource_bundle";
1750
1751 foreach my $interface_file_key (@gs3_interface_files) {
1752
1753 &log_message("Greenstone 3 interface file: " . $interface_file_key);
1754
1755 # Parse the source language and target language files
1756 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1757 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1758 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1759 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1760 my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1761
1762 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1763 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1764 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1765 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1766 my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1767
1768
1769 # Filter out any automatically translated chunks
1770 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1771 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1772 delete $source_file_key_to_line_mapping{$chunk_key};
1773 delete $target_file_key_to_line_mapping{$chunk_key};
1774 }
1775 }
1776
1777 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1778 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1779
1780 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1781 my $global_chunk_key = "$interface_file_key.$chunk_key";
1782 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1783 $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1784
1785 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1786 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1787 $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1788 }
1789 }
1790 }
1791}
1792
1793
1794sub write_translated_gs3interface
1795{
1796 my $source_file_key_to_text_mapping = shift(@_);
1797 my $target_file_key_to_text_mapping = shift(@_);
1798 my $target_file_key_to_gti_comment_mapping = shift(@_);
1799 my $target_language_code = shift(@_);
1800
1801 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1802
1803 my %translated_interface_file_keys = ();
1804 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1805 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1806 if (!defined $translated_interface_file_keys{$1}) {
1807 &log_message("Updated interface file: " . $1);
1808 $translated_interface_file_keys{$1}="";
1809 }
1810 }
1811 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1812
1813 my $source_file_directory = "greenstone3";
1814
1815 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1816
1817 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1818 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1819 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1820 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1821 my %source_file_line_to_key_mapping = ();
1822 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1823 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1824 }
1825
1826 # Write the new target file
1827 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1828 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1829 if (!open(TARGET_FILE, ">$target_file_path")) {
1830 &throw_fatal_error("Could not write target file $target_file_path.");
1831 }
1832
1833 # Model the new target file on the source file, with the target file translations
1834 my $source_file_line_number = 0;
1835 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1836 # Fill in the gaps before this chunk starts
1837 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1838 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1839 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1840 print TARGET_FILE $source_file_lines[$source_file_line_number];
1841 $source_file_line_number++;
1842 }
1843 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1844
1845 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1846 my $global_chunk_key = "$interface_file_key.$chunk_key";
1847 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1848 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1849
1850 # If no translation exists for this chunk, show this, and move on
1851 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1852 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1853 next;
1854 }
1855
1856 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1857 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1858 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1859 }
1860 print TARGET_FILE "\n";
1861 }
1862
1863 close(TARGET_FILE);
1864 }
1865}
1866
1867&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.