source: main/trunk/greenstone2/bin/script/gti.pl@ 24093

Last change on this file since 24093 was 24093, checked in by sjm84, 13 years ago

Fixing issues with perl finding the wrong perl by making sure it uses the one that is currently running
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 77.1 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33
34 if(!$ENV{'PERLPATH'}) {
35 my $full_perl_exec = $^X;
36 require File::Basename;
37 my $perl_path = File::Basename::dirname($full_perl_exec);
38 $ENV{'PERLPATH'} = $perl_path;
39 }
40}
41
42use iso639;
43use strict;
44use util;
45
46
47#my $anonymous_cvs_root = ":pserver:cvs_anon\@cvs.scms.waikato.ac.nz:2402/usr/local/global-cvs/gsdl-src";
48#my $anonymous_svn_root = "http://http://svn.greenstone.org/gsdl/trunk/";
49my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
50my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
51my $source_language_code = "en"; # This is non-negiotable
52
53my $gti_translation_files =
54 [ # Greenstone macrofiles
55 { 'key' => "coredm",
56 'file_type' => "macrofile",
57 'source_file' => "macros/english.dm",
58 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
59 { 'key' => "auxdm",
60 'file_type' => "macrofile",
61 'source_file' => "macros/english2.dm",
62 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
63
64 # GLI dictionary
65 { 'key' => "glidict",
66 'file_type' => "resource_bundle",
67 'source_file' => "gli/classes/dictionary.properties",
68 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
69
70 # GLI help
71 { 'key' => "glihelp",
72 'file_type' => "greenstone_xml",
73 'source_file' => "gli/help/en/help.xml",
74 'target_file' => "gli/help/{target_language_code}/help.xml" },
75
76 # Greenstone Perl modules
77 { 'key' => "perlmodules",
78 'file_type' => "resource_bundle",
79 'source_file' => "perllib/strings.properties",
80 'target_file' => "perllib/strings_{target_language_code}.properties" },
81
82 # Greenstone tutorial exercises
83 # { 'key' => "tutorials",
84 # 'file_type' => "greenstone_xml",
85 # 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
86 # 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
87
88 # new Greenstone.org
89 { 'key' => "greenorg",
90 'file_type' => "resource_bundle",
91 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
92 'target_file' => "greenstoneorg/website/classes/Gsc_{iso_639_1_target_language_name}.properties"
93 # 'file_type' => "macrofile",
94 # 'source_file' => "greenorg/macros/english.dm",
95 # 'target_file' => "greenorg/macros/{iso_639_1_target_language_name}.dm"
96 }
97
98 # { 'key' => "gs3interface",
99 #'file_type' => "resource_bundle",
100 #'source_file' => "greenstone3",
101 #'target_file' => "greenstone3"
102 #}
103 ];
104
105my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "Authentication", "CrossCollectionSearch", "GATEServices", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_nzdl", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "QBRWebServicesHelp", "Visualizer");
106
107
108sub main
109{
110 # Get the command to process, and any arguments
111 my $gti_command = shift(@_);
112 my @gti_command_arguments = @_;
113 my $module = $_[1];
114
115 # Open the GTI log file for appending, or write to STDERR if that fails
116 if (!open(GTI_LOG, ">>$gti_log_file")) {
117 open(GTI_LOG, ">&STDERR");
118 }
119
120 # Log the command that launched this script
121 &log_message("Command: $0 @ARGV");
122
123 # Check that a command was supplied
124 if (!$gti_command) {
125 &throw_fatal_error("Missing command.");
126 }
127
128 # Process the command
129 if ($gti_command =~ /^get-all-chunks$/i) {
130 # Check that GS3 interface is the target
131 if ($module eq "gs3interface") {
132 print &get_all_chunks_for_gs3(@gti_command_arguments);
133 } else {
134 print &get_all_chunks(@gti_command_arguments);
135 }
136 }
137 if ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
138 if ($module eq "gs3interface") {
139 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
140 } else {
141 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
142 }
143 }
144 if ($gti_command =~ /^get-language-status$/i) {
145 print &get_language_status(@gti_command_arguments);
146 }
147 if ($gti_command =~ /^search-chunks$/i) {
148 print &search_chunks(@gti_command_arguments);
149 }
150 if ($gti_command =~ /^submit-translations$/i) {
151 # This command cannot produce any output since it reads input
152 &submit_translations(@gti_command_arguments);
153 }
154 if ($gti_command =~ /^create-glihelp-zip-file$/i) {
155 # This command cannot produce any output since it reads input
156 &create_glihelp_zip_file(@gti_command_arguments);
157 }
158# else {
159 # The command was not recognized
160 #&throw_fatal_error("Unknown command \"$gti_command\".");
161 #}
162}
163
164
165sub throw_fatal_error
166{
167 my $error_message = shift(@_);
168
169 # Write an XML error response
170 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
171 print "<GTIResponse>\n";
172 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
173 print "</GTIResponse>\n";
174
175 # Log the error message, then die
176 &log_message("Error: $error_message");
177 die "\n";
178}
179
180
181sub log_message
182{
183 my $log_message = shift(@_);
184 print GTI_LOG time() . " -- " . $log_message . "\n";
185}
186
187
188sub get_all_chunks
189{
190 # The code of the target language (ensure it is lowercase)
191 my $target_language_code = lc(shift(@_));
192 # The key of the file to translate (ensure it is lowercase)
193 my $translation_file_key = lc(shift(@_));
194
195 # Check that the necessary arguments were supplied
196 if (!$target_language_code || !$translation_file_key) {
197 &throw_fatal_error("Missing command argument.");
198 }
199
200 # Get (and check) the translation configuration
201 my ($source_file, $target_file, $translation_file_type)
202 = &get_translation_configuration($target_language_code, $translation_file_key);
203
204 # Parse the source language and target language files
205 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
206 my @source_file_lines = &read_file_lines($source_file_path);
207 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
208
209 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
210 my @target_file_lines = &read_file_lines($target_file_path);
211 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
212
213 # Filter out any automatically translated chunks
214 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
215 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
216 delete $source_file_key_to_line_mapping{$chunk_key};
217 delete $target_file_key_to_line_mapping{$chunk_key};
218 }
219 }
220
221 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
222 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
223 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
224 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
225
226 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
227 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
228
229 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
230
231 return $xml_response;
232}
233
234
235sub get_first_n_chunks_requiring_work
236{
237 # The code of the target language (ensure it is lowercase)
238 my $target_language_code = lc(shift(@_));
239 # The key of the file to translate (ensure it is lowercase)
240 my $translation_file_key = lc(shift(@_));
241 # The number of chunks to return (defaults to one if not specified)
242 my $num_chunks_to_return = shift(@_) || "1";
243
244 # Check that the necessary arguments were supplied
245 if (!$target_language_code || !$translation_file_key) {
246 &throw_fatal_error("Missing command argument.");
247 }
248
249 # Get (and check) the translation configuration
250 my ($source_file, $target_file, $translation_file_type)
251 = &get_translation_configuration($target_language_code, $translation_file_key);
252
253 # Parse the source language and target language files
254 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
255 my @source_file_lines = &read_file_lines($source_file_path);
256 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
257
258 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
259 my @target_file_lines = &read_file_lines($target_file_path);
260 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
261
262 # Filter out any automatically translated chunks
263 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
264 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
265 delete $source_file_key_to_line_mapping{$chunk_key};
266 delete $target_file_key_to_line_mapping{$chunk_key};
267 }
268 }
269
270 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
271 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
272 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
273 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
274
275 # Determine the target file chunks requiring translation
276 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
277 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
278
279 # Determine the target file chunks requiring updating
280 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
281 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
282 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
283 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
284
285 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
286
287 return $xml_response;
288}
289
290
291sub get_language_status
292{
293 # The code of the target language (ensure it is lowercase)
294 my $target_language_code = lc(shift(@_));
295
296 # Check that the necessary arguments were supplied
297 if (!$target_language_code) {
298 &throw_fatal_error("Missing command argument.");
299 }
300
301 # Form an XML response to the command
302 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
303 $xml_response .= "<GTIResponse>\n";
304 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
305
306 foreach my $translation_file (@$gti_translation_files) {
307 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
308 my $target_file_name = "";
309
310 if ($translation_file->{'key'} eq "gs3interface") {
311 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
312 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
313
314 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
315 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
316
317 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
318 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
319 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
320 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
321 }
322 else {
323 # Get (and check) the translation configuration
324 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
325 $target_file_name = $target_file;
326
327 # Parse the source language and target language files
328 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
329 my @source_file_lines = &read_file_lines($source_file_path);
330 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
331
332 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
333 my @target_file_lines = &read_file_lines($target_file_path);
334 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
335
336 # Filter out any automatically translated chunks
337 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
338 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
339 delete $source_file_key_to_line_mapping{$chunk_key};
340 delete $target_file_key_to_line_mapping{$chunk_key};
341 }
342 }
343
344 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
345 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
346
347 # Determine the target file chunks requiring translation
348 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
349
350 # Determine the target file chunks requiring updating
351 my @target_file_keys_requiring_updating = ();
352 if (-e $target_file_path) {
353 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
354 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
355 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
356 }
357
358 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
359 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
360 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
361 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
362 }
363
364 &log_message("Status of " . $translation_file->{'key'});
365 &log_message("Number of source chunks: " . $num_source_chunks);
366 &log_message("Number of target chunks: " . $num_target_chunks);
367 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
368 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
369
370 $xml_response .= " <TranslationFile"
371 . " key=\"" . $translation_file->{'key'} . "\""
372 . " target_file_path=\"" . $target_file_name . "\""
373 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
374 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
375 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
376 }
377
378 $xml_response .= " </LanguageStatus>\n";
379
380 $xml_response .= "</GTIResponse>\n";
381 return $xml_response;
382}
383
384
385sub search_chunks
386{
387 # The code of the target language (ensure it is lowercase)
388 my $target_language_code = lc(shift(@_));
389 # The key of the file to translate (ensure it is lowercase)
390 my $translation_file_key = lc(shift(@_));
391 # The query string
392 my $query_string = join(' ', @_);
393
394 # Check that the necessary arguments were supplied
395 if (!$target_language_code || !$translation_file_key || !$query_string) {
396 &throw_fatal_error("Missing command argument.");
397 }
398
399 my ($source_file, $target_file, $translation_file_type) = ();
400 my %source_file_key_to_text_mapping = ();
401 my %target_file_key_to_text_mapping = ();
402
403
404 if ($translation_file_key ne "gs3interface") {
405 # Get (and check) the translation configuration
406 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
407
408 # Parse the source language and target language files
409 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
410 my @source_file_lines = &read_file_lines($source_file_path);
411 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
412
413 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
414 my @target_file_lines = &read_file_lines($target_file_path);
415 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
416
417 # Filter out any automatically translated chunks
418 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
419 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
420 delete $source_file_key_to_line_mapping{$chunk_key};
421 delete $target_file_key_to_line_mapping{$chunk_key};
422 }
423 }
424
425 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
426 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
427 }
428 else {
429 # Not needed in this case
430 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
431 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
432 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
433 }
434
435 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
436 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
437
438 # Determine the target file chunks matching the query
439 my @target_file_keys_matching_query = ();
440 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
441 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
442 if ($target_file_text =~ /$query_string/i) {
443 # &log_message("Chunk with key $chunk_key matches query.");
444 push(@target_file_keys_matching_query, $chunk_key);
445 }
446 }
447
448 # Form an XML response to the command
449 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
450 $xml_response .= "<GTIResponse>\n";
451
452 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
453 foreach my $chunk_key (@target_file_keys_matching_query) {
454 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
455
456 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
457 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
458 $xml_response .= " </Chunk>\n";
459 }
460 $xml_response .= " </ChunksMatchingQuery>\n";
461
462 $xml_response .= "</GTIResponse>\n";
463 return $xml_response;
464}
465
466
467sub submit_translations
468{
469 # The code of the target language (ensure it is lowercase)
470 my $target_language_code = lc(shift(@_));
471 # The key of the file to translate (ensure it is lowercase)
472 my $translation_file_key = lc(shift(@_));
473 # The username of the translation submitter
474 my $submitter_username = shift(@_);
475 # Whether to submit a target chunk even if it hasn't changed
476 my $force_submission_flag = shift(@_);
477
478 # Check that the necessary arguments were supplied
479 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
480 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
481 die "\n";
482 }
483
484 my %source_file_key_to_text_mapping = ();
485 my %source_file_key_to_gti_comment_mapping = ();
486 my %target_file_key_to_text_mapping = ();
487 my %target_file_key_to_gti_comment_mapping = ();
488
489 my (@source_file_lines, @target_file_lines) = ();
490 my ($source_file, $target_file, $translation_file_type);
491
492
493 if ($translation_file_key ne "gs3interface") {
494 # Get (and check) the translation configuration
495 ($source_file, $target_file, $translation_file_type)
496 = &get_translation_configuration($target_language_code, $translation_file_key);
497
498 # Parse the source language and target language files
499 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
500 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
501 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
502 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
503
504 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
505 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
506 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
507 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
508 }
509 else {
510 &build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
511 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
512 }
513 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
514 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
515
516 # Submission date
517 my $day = (localtime)[3];
518 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
519 my $year = (localtime)[5] + 1900;
520 my $submission_date = "$day-$month-$year";
521
522 open(SUBMISSION, "-");
523 my @submission_lines = <SUBMISSION>;
524 close(SUBMISSION);
525
526 # Remove any nasty carriage returns
527 # &log_message("Submission:");
528 foreach my $submission_line (@submission_lines) {
529 $submission_line =~ s/\r$//;
530 #&log_message(" $submission_line");
531 }
532
533 my %source_file_key_to_submission_mapping = ();
534 my %target_file_key_to_submission_mapping = ();
535 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
536 # Read source file part of submission
537 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
538 my $chunk_key = $1;
539
540 # Read the source file text
541 my $source_file_chunk_text = "";
542 $i++;
543 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
544 $source_file_chunk_text .= $submission_lines[$i];
545 $i++;
546 }
547 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
548 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
549
550 #&log_message("Source file key: $chunk_key");
551 #&log_message("Source file text: $source_file_chunk_text");
552 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
553 }
554
555 # Read target file part of submission
556 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
557 my $chunk_key = $1;
558
559 # Read the target file text
560 my $target_file_chunk_text = "";
561 $i++;
562 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
563 $target_file_chunk_text .= $submission_lines[$i];
564 $i++;
565 }
566 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
567 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
568
569 #&log_message("Target file key: $chunk_key");
570 #&log_message("Target file text: $target_file_chunk_text");
571 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
572 }
573 }
574
575 # -----------------------------------------
576 # Validate the translation submissions
577 # -----------------------------------------
578
579 # Check that the translations are valid
580 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
581 # Make sure the submitted chunk still exists in the source file
582 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
583 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
584 delete $source_file_key_to_submission_mapping{$chunk_key};
585 delete $target_file_key_to_submission_mapping{$chunk_key};
586 next;
587 }
588
589 # Make sure the submitted source chunk matches the source file chunk
590 if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
591 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
592 &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
593 &log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
594 delete $source_file_key_to_submission_mapping{$chunk_key};
595 delete $target_file_key_to_submission_mapping{$chunk_key};
596 next;
597 }
598 }
599
600 # Apply the submitted translations
601 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
602 # Only apply the submission if it is a change, unless -force_submission has been specified
603 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
604 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
605 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
606 }
607 }
608
609 if ($translation_file_key ne "gs3interface") {
610 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
611 } else {
612 eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
613 }
614}
615
616
617sub create_glihelp_zip_file
618{
619 my $target_language_code = shift(@_);
620 my $translation_file_key = "glihelp";
621
622 &log_message("Creating GLI Help zip file for $target_language_code");
623
624 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
625
626 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
627 if ( ! -e $classpath) {
628 &throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
629 }
630
631 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
632 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
633
634 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
635 if ( ! -e $gen_many_html_xsl_filepath) {
636 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
637 }
638
639 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
640 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
641
642 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
643 $target_file_directory = $target_file_directory."/";
644
645 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
646 my $perl_exec = &util::filename_cat($ENV{'PERLPATH'}, "perl");
647 my $java_exec = "java";
648 if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
649 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
650 }
651
652 my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | $perl_exec -S $split_script_filepath $target_file_directory";
653 my $response = `$cmd`;
654 &log_message($cmd);
655 &log_message($response);
656 &log_message("Created HTML operational files");
657
658 $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
659 $response = `$cmd`;
660 &log_message($cmd);
661 &log_message($response);
662 &log_message("Created xml index file");
663
664 my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
665 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
666 $response = `$cmd`;
667 &log_message($cmd);
668 &log_message($response);
669 &log_message("Created the zip file");
670}
671
672
673sub get_translation_configuration
674{
675 # Get the code of the target language
676 my $target_language_code = shift(@_);
677 # Get the key of the file to translate
678 my $translation_file_key = shift(@_);
679
680 # Read the translation data from the gti.cfg file
681 my ($source_file, $target_file, $translation_file_type) =
682 &get_translation_data_for($target_language_code, $translation_file_key);
683
684 # Check that the file to translate is defined in the gti.cfg file
685 if (!$source_file || !$target_file || !$translation_file_type) {
686 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
687 }
688
689 # Check that the source file exists
690 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
691 if (!-e $source_file_path) {
692 &throw_fatal_error("Source file $source_file_path does not exist.");
693 }
694
695 # Check that the source file is up to date
696 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
697 # unless ($translation_file_is_not_in_cvs) {
698 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
699 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
700 if ($source_file_cvs_status =~ /^C /) {
701 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
702 }
703 if ($source_file_cvs_status =~ /^M /) {
704 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
705 }
706 # }
707
708 return ($source_file, $target_file, $translation_file_type);
709}
710
711
712sub get_translation_data_for
713{
714 my ($target_language_code, $translation_file_key) = @_;
715
716 foreach my $translation_file (@$gti_translation_files) {
717 # If this isn't the correct translation file, move onto the next one
718 next if ($translation_file_key ne $translation_file->{'key'});
719
720 # Resolve the target language file
721 my $target_language_file = $translation_file->{'target_file'};
722 if ($target_language_file =~ /(\{.+\;.+\})/) {
723 my $unresolved_target_language_file_part = $1;
724
725 # Check for a special case for the target language code
726 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
727 my $resolved_target_language_file_part = $2;
728 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
729 }
730 # Otherwise use the last part as the default value
731 else {
732 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
733 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
734 }
735 }
736
737 # Resolve instances of {iso_639_1_target_language_name}
738 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
739 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
740 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
741
742 # Resolve instances of {target_language_code}
743 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
744
745 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
746 }
747
748 return ();
749}
750
751
752sub read_file_lines
753{
754 my ($file_path) = @_;
755
756 if (!open(FILE_IN, "<$file_path")) {
757 &log_message("Note: Could not open file $file_path.");
758 return ();
759 }
760 my @file_lines = <FILE_IN>;
761 close(FILE_IN);
762
763 return @file_lines;
764}
765
766
767sub build_key_to_line_mapping
768{
769 my ($file_lines, $translation_file_type) = @_;
770 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
771}
772
773
774sub build_key_to_text_mapping
775{
776 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
777
778 my %key_to_text_mapping = ();
779 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
780 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
781 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
782
783 my $chunk_text = @$file_lines[$chunk_starting_line];
784 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
785 $chunk_text .= @$file_lines[$l];
786 }
787
788 # Map from chunk key to text
789 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
790 }
791
792 return %key_to_text_mapping;
793}
794
795
796sub build_key_to_last_update_date_mapping
797{
798 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
799
800 # If the files aren't in CVS then we can't tell anything about what needs updating
801 # return () if ($translation_file_is_not_in_cvs);
802
803 # Build a mapping from key to CVS date
804 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
805 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
806
807 # Build a mapping from key to comment date
808 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
809
810 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
811 my %key_to_last_update_date_mapping = ();
812 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
813 # Use the CVS date as a starting point
814 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
815 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
816
817 # If a comment date exists and it is after the CVS date, use that instead
818 # need to convert the comment date format to SVN format
819 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
820 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
821 my $chunk_comment_date = $1;
822 # if the changes were made on 2009 Jan 27, ie. tidying up the GTI commands following each fragment,
823 # then should use the GTI comment date instead
824 # if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
825 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date)
826 || (&is_date_after($chunk_cvs_date,$chunk_comment_date) && ($chunk_cvs_date eq "2009-01-27" || $chunk_cvs_date eq "2009-01-28")))) {
827 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
828
829 # &log_message("cvs date: $chunk_cvs_date, comment date: $chunk_comment_date");
830 # die;
831 }
832 }
833 }
834
835 return %key_to_last_update_date_mapping;
836}
837
838
839sub build_key_to_cvs_date_mapping
840{
841 my ($filename, $translation_file_type) = @_;
842
843 # Use CVS to annotate each line of the file with the date it was last edited
844 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
845 # my $cvs_annotated_file = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root annotate -F $filename 2>/dev/null`;
846 # my $cvs_annotated_file = `cd $gsdl_root_directory; export PATH=.:/research/lh92/programs/subversion/bin; svn annotate -v --force $filename`;
847 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v $filename`;
848
849 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
850
851 my @cvs_annotated_file_lines_date = ();
852 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
853 # Extract the date from the CVS annotation at the front
854 # cvs format : 07-Jun-02
855 # svn format : 2007-07-16
856 # $cvs_annotated_file_line =~ s/^\S+\s+\(\S+\s+(\S+)\):\s//;
857 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
858
859 push(@cvs_annotated_file_lines_date, $1);
860
861 # trim extra date information in svn annotation format
862 # 15:42:49 +1200 (Wed, 21 Jun 2006)
863 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
864 }
865
866 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
867 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
868
869 my %key_to_cvs_date_mapping = ();
870 foreach my $chunk_key (keys(%key_to_line_mapping)) {
871 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
872 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
873
874 # Find the date this chunk was last edited, from the CVS annotation
875 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
876 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
877 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
878 # This part of the chunk has been updated more recently
879 $chunk_date = $cvs_annotated_file_lines_date[$l];
880
881 }
882 }
883
884 # Map from chunk key to CVS date
885 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
886 }
887
888 return %key_to_cvs_date_mapping;
889}
890
891
892sub build_key_to_gti_comment_mapping
893{
894 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
895
896 my %key_to_gti_comment_mapping = ();
897 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
898 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
899 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
900
901 my $chunk_text = @$file_lines[$chunk_starting_line];
902 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
903 $chunk_text .= @$file_lines[$l];
904 }
905
906 # Map from chunk key to GTI comment
907 my $chunk_gti_comment;
908 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
909 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
910 }
911
912 return %key_to_gti_comment_mapping;
913}
914
915
916sub determine_chunks_requiring_translation
917{
918 my $source_file_key_to_text_mapping = shift(@_);
919 my $target_file_key_to_text_mapping = shift(@_);
920
921 # Chunks needing translation are those in the source file with no translation in the target file
922 my @target_file_keys_requiring_translation = ();
923 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
924 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
925 # &log_message("Chunk with key $chunk_key needs translating.");
926 push(@target_file_keys_requiring_translation, $chunk_key);
927 }
928 }
929
930 return @target_file_keys_requiring_translation;
931}
932
933
934sub determine_chunks_requiring_updating
935{
936 my $source_file_key_to_last_update_date_mapping = shift(@_);
937 my $target_file_key_to_last_update_date_mapping = shift(@_);
938
939 # Chunks needing updating are those in the target file that have been more recently edited in the source file
940 my @target_file_keys_requiring_updating = ();
941 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
942 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
943 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
944
945 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
946
947 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
948 # &log_message("Chunk with key $chunk_key needs updating.");
949 push(@target_file_keys_requiring_updating, $chunk_key);
950 }
951 }
952
953 return @target_file_keys_requiring_updating;
954}
955
956
957sub is_chunk_automatically_translated
958{
959 my ($chunk_key, $translation_file_type) = @_;
960 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
961}
962
963
964sub make_text_xml_safe
965{
966 my $text = shift(@_);
967 $text =~ s/\&/\&amp\;/g;
968 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
969 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
970 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
971 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
972 $text =~ s/</\&lt\;/g;
973 $text =~ s/>/\&gt\;/g;
974 return $text;
975}
976
977
978sub unmake_text_xml_safe
979{
980 my $text = shift(@_);
981 $text =~ s/\&lt\;/</g;
982 $text =~ s/\&gt\;/>/g;
983 $text =~ s/\&amp\;/\&/g;
984 return $text;
985}
986
987
988# Returns 1 if $date1 is after $date2, 0 otherwise
989sub is_date_after_cvs
990{
991 my ($date1, $date2) = @_;
992 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
993 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
994
995 if(!defined $date1) {
996 return 1;
997 }
998
999 my @date1parts = split(/-/, $date1);
1000 my @date2parts = split(/-/, $date2);
1001
1002 # Compare year - nasty because we have rolled over into a new century
1003 my $year1 = $date1parts[2];
1004 if ($year1 < 80) {
1005 $year1 += 2000;
1006 }
1007 my $year2 = $date2parts[2];
1008 if ($year2 < 80) {
1009 $year2 += 2000;
1010 }
1011
1012 # Compare year
1013 if ($year1 > $year2) {
1014 return 1;
1015 }
1016 elsif ($year1 == $year2) {
1017 # Year is the same, so compare month
1018 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1019 return 1;
1020 }
1021 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1022 # Month is the same, so compare day
1023 if ($date1parts[0] > $date2parts[0]) {
1024 return 1;
1025 }
1026 }
1027 }
1028
1029 return 0;
1030}
1031
1032sub is_date_after
1033{
1034 my ($date1, $date2) = @_;
1035
1036 if(!defined $date1) {
1037 return 1;
1038 }
1039 if(!defined $date2) {
1040 return 0;
1041 }
1042
1043 # 16-Aug-2006
1044 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1045 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1046 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1047 $date1=$3 . "-" . $months{$2} . "-" . $1;
1048 # print "** converted date1: $date1\n";
1049 }
1050 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1051 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1052 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1053 $date2=$3 . "-" . $months{$2} . "-" . $1;
1054 # print "** converted date2: $date2\n";
1055 }
1056
1057
1058 # 2006-08-16
1059 my @date1parts = split(/-/, $date1);
1060 my @date2parts = split(/-/, $date2);
1061
1062 # Compare year
1063 if ($date1parts[0] > $date2parts[0]) {
1064 return 1;
1065 }
1066 elsif ($date1parts[0] == $date2parts[0]) {
1067 # Year is the same, so compare month
1068 if ($date1parts[1] > $date2parts[1]) {
1069 return 1;
1070 }
1071 elsif ($date1parts[1] == $date2parts[1]) {
1072 # Month is the same, so compare day
1073 if ($date1parts[2] > $date2parts[2]) {
1074 return 1;
1075 }
1076 }
1077 }
1078
1079 return 0;
1080}
1081
1082
1083sub create_xml_response_for_chunks_requiring_work
1084{
1085 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1086
1087 # Form an XML response to the command
1088 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1089 $xml_response .= "<GTIResponse>\n";
1090 $xml_response .= " <TranslationFile"
1091 . " key=\"" . $translation_file_key . "\""
1092 . " target_file_path=\"" . $target_file . "\""
1093 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1094 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1095 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1096
1097 # Do chunks requiring translation first
1098 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1099 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1100 }
1101 else {
1102 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1103 }
1104
1105 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1106 foreach my $chunk_key (@sorted_chunk_keys) {
1107 last if ($num_chunks_to_return == 0);
1108
1109 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1110 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1111
1112 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1113 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1114 $xml_response .= " <TargetFileText></TargetFileText>\n";
1115 $xml_response .= " </Chunk>\n";
1116
1117 $num_chunks_to_return--;
1118 }
1119
1120 $xml_response .= " </ChunksRequiringTranslation>\n";
1121
1122 # Then do chunks requiring updating
1123 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1124 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1125 }
1126 else {
1127 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1128 }
1129
1130 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1131 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1132 foreach my $chunk_key (@sorted_chunk_keys) {
1133 last if ($num_chunks_to_return == 0);
1134
1135 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1136 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1137 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1138 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1139
1140 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1141 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1142 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1143 $xml_response .= " </Chunk>\n";
1144
1145 $num_chunks_to_return--;
1146 }
1147
1148 $xml_response .= " </ChunksRequiringUpdating>\n";
1149
1150 $xml_response .= "</GTIResponse>\n";
1151
1152 return $xml_response;
1153}
1154
1155
1156sub create_xml_response_for_all_chunks
1157{
1158 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1159
1160 # Form an XML response to the command
1161 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1162 $xml_response .= "<GTIResponse>\n";
1163 $xml_response .= " <TranslationFile"
1164 . " key=\"" . $translation_file_key . "\""
1165 . " target_file_path=\"" . $target_file . "\"\/>\n";
1166
1167 # Do all the chunks
1168 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1169
1170 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1171 foreach my $chunk_key (@sorted_chunk_keys) {
1172 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1173 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1174
1175 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1176 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1177 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1178 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1179 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1180 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1181 }
1182 else {
1183 $xml_response .= " <TargetFileText></TargetFileText>\n";
1184 }
1185
1186 $xml_response .= " </Chunk>\n";
1187 }
1188 $xml_response .= " </Chunks>\n";
1189
1190 $xml_response .= "</GTIResponse>\n";
1191 return $xml_response;
1192}
1193
1194
1195
1196# ==========================================================================================
1197# MACROFILE FUNCTIONS
1198
1199sub build_key_to_line_mapping_for_macrofile
1200{
1201 my (@file_lines) = @_;
1202
1203 my $macro_package;
1204 my %chunk_key_to_line_mapping = ();
1205 # Process the contents of the file, line by line
1206 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1207 my $line = $file_lines[$i];
1208 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1209
1210 # Check if a new package is being defined
1211 if ($line =~ m/^package\s+(.+)/) {
1212 $macro_package = $1;
1213 }
1214
1215 # Line contains a macro name
1216 elsif ($line =~ m/^(_\w+_)/) {
1217 my $macro_key = $1;
1218 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1219
1220 # While there is still text of the macro to go...
1221 my $startline = $i;
1222 while ($line !~ /\}$/) {
1223 $i++;
1224 if ($i == scalar(@file_lines)) {
1225 &throw_fatal_error("Could not find end of macro $macro_key.");
1226 }
1227 $line = $file_lines[$i];
1228 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1229 }
1230
1231 # The chunk key consists of the package name and the macro key
1232 my $chunk_key = $macro_package . "." . $macro_key;
1233 # Map from chunk key to line
1234 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1235 }
1236
1237 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1238 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1239 # The chunk key consists of package name and macro key
1240 my $chunk_key = $macro_package . "." . $1;
1241 # Map from chunk key to line
1242 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1243 }
1244 }
1245
1246 return %chunk_key_to_line_mapping;
1247}
1248
1249
1250sub import_chunk_from_macrofile
1251{
1252 my ($chunk_text) = @_;
1253
1254 # Is this an icon macro??
1255 if ($chunk_text =~ /^\#\# (.*)/) {
1256 # Extract image macro text
1257 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1258 $chunk_text = $1;
1259
1260 # Remove enclosing quotes
1261 $chunk_text =~ s/^\"//;
1262 $chunk_text =~ s/\"$//;
1263 }
1264
1265 # No, so it must be a text macro
1266 else {
1267 # Remove macro key
1268 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1269
1270 # Remove language specifier
1271 $chunk_text =~ s/^\[l=.*\](\s*)//;
1272
1273 # Remove braces enclosing text
1274 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1275 }
1276
1277 return $chunk_text;
1278}
1279
1280
1281sub get_macrofile_chunk_gti_comment
1282{
1283 my ($chunk_text) = @_;
1284
1285 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1286 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1287 return $1;
1288 }
1289
1290 return undef;
1291}
1292
1293
1294sub is_macrofile_chunk_automatically_translated
1295{
1296 my ($chunk_key) = @_;
1297
1298 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1299 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1300 return 1;
1301 }
1302
1303 return 0;
1304}
1305
1306
1307# Use the source file to generate a target file that is formatted the same
1308sub write_translated_macrofile
1309{
1310 my $source_file = shift(@_); # Not used
1311 my @source_file_lines = @{shift(@_)};
1312 my $source_file_key_to_text_mapping = shift(@_);
1313 my $target_file = shift(@_);
1314 my @target_file_lines = @{shift(@_)};
1315 my $target_file_key_to_text_mapping = shift(@_);
1316 my $target_file_key_to_gti_comment_mapping = shift(@_);
1317 my $target_language_code = shift(@_);
1318
1319 # Build a mapping from source file line to chunk key
1320 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1321 my %source_file_line_to_key_mapping = ();
1322 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1323 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1324 }
1325 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1326 my $source_file_line_number = 0;
1327
1328 # Build a mapping from target file line to chunk key
1329 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1330 my %target_file_line_to_key_mapping = ();
1331 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1332 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1333 }
1334 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1335
1336 # Write the new target file
1337 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1338 if (!open(TARGET_FILE, ">$target_file_path")) {
1339 &throw_fatal_error("Could not write target file $target_file_path.");
1340 }
1341
1342 # Use the header from the target file, to keep language and author information
1343 if (scalar(@target_file_line_keys) > 0) {
1344 my $target_file_line_number = 0;
1345 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1346 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1347 my $target_file_line = $target_file_lines[$target_file_line_number];
1348 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1349 print TARGET_FILE $target_file_line;
1350 $target_file_line_number++;
1351 }
1352
1353 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1354 }
1355
1356 # Model the new target file on the source file, with the target file translations
1357 foreach my $line_key (@source_file_line_keys) {
1358 # Fill in the gaps before this chunk starts
1359 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1360 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1361 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1362 print TARGET_FILE $source_file_lines[$source_file_line_number];
1363 $source_file_line_number++;
1364 }
1365 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1366
1367 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1368 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1369 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1370
1371 my $macrofile_key = $chunk_key;
1372 $macrofile_key =~ s/^(.+?)\.//;
1373
1374 # If no translation exists for this chunk, show this, and move on
1375 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1376 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1377 next;
1378 }
1379
1380 # Grab the source chunk text
1381 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1382 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1383 $source_file_chunk .= $source_file_lines[$l];
1384 }
1385
1386 # Is this an icon macro??
1387 if ($source_file_chunk =~ /^\#\# (.*)/) {
1388 # Escape any newline and question mark characters so the source text is replaced correctly
1389 $source_file_chunk_text =~ s/\\/\\\\/g;
1390 $source_file_chunk_text =~ s/\?/\\\?/g;
1391
1392 # Build the new target chunk from the source chunk
1393 my $target_file_chunk = $source_file_chunk;
1394 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1395 $target_file_chunk =~ s/(\s)*$//;
1396 print TARGET_FILE "$target_file_chunk";
1397 }
1398
1399 # No, it is just a normal text macro
1400 else {
1401 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1402 }
1403
1404 # Add the "updated" comment, if one exists
1405 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1406 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1407 }
1408 print TARGET_FILE "\n";
1409 }
1410
1411 close(TARGET_FILE);
1412}
1413
1414
1415sub sort_by_line
1416{
1417 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1418}
1419
1420
1421# ==========================================================================================
1422# RESOURCE BUNDLE FUNCTIONS
1423
1424sub build_key_to_line_mapping_for_resource_bundle
1425{
1426 my (@file_lines) = @_;
1427
1428 my %chunk_key_to_line_mapping = ();
1429 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1430 my $line = $file_lines[$i];
1431 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1432
1433 # Line contains a dictionary string
1434 if ($line =~ /^(\S+?)[:|=](.*)$/) {
1435 my $chunk_key = $1;
1436
1437 # Map from chunk key to line
1438 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1439 }
1440 }
1441
1442 return %chunk_key_to_line_mapping;
1443}
1444
1445
1446sub import_chunk_from_resource_bundle
1447{
1448 my ($chunk_text) = @_;
1449
1450 # Simple: just remove string key
1451 $chunk_text =~ s/^(\S+?)[:|=](\s*)//;
1452 $chunk_text =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1453 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//i;
1454
1455 return $chunk_text;
1456}
1457
1458
1459sub get_resource_bundle_chunk_gti_comment
1460{
1461 my ($chunk_text) = @_;
1462
1463 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1464 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1465 return $1;
1466 }
1467
1468 return undef;
1469}
1470
1471
1472sub is_resource_bundle_chunk_automatically_translated
1473{
1474 # No resource bundle chunks are automatically translated
1475 return 0;
1476}
1477
1478
1479sub write_translated_resource_bundle
1480{
1481 my $source_file = shift(@_); # Not used
1482 my @source_file_lines = @{shift(@_)};
1483 my $source_file_key_to_text_mapping = shift(@_);
1484 my $target_file = shift(@_);
1485 my @target_file_lines = @{shift(@_)}; # Not used
1486 my $target_file_key_to_text_mapping = shift(@_);
1487 my $target_file_key_to_gti_comment_mapping = shift(@_);
1488 my $target_language_code = shift(@_); # Not used
1489
1490 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1491 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1492 my %source_file_line_to_key_mapping = ();
1493 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1494 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1495 }
1496
1497 # Write the new target file
1498 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1499 if (!open(TARGET_FILE, ">$target_file_path")) {
1500 &throw_fatal_error("Could not write target file $target_file_path.");
1501 }
1502
1503 # Model the new target file on the source file, with the target file translations
1504 my $source_file_line_number = 0;
1505 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1506 # Fill in the gaps before this chunk starts
1507 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1508 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1509 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1510 print TARGET_FILE $source_file_lines[$source_file_line_number];
1511 $source_file_line_number++;
1512 }
1513 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1514
1515 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1516 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1517 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1518
1519 # If no translation exists for this chunk, show this, and move on
1520 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1521 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1522 next;
1523 }
1524
1525 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1526 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1527 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1528 }
1529 print TARGET_FILE "\n";
1530 }
1531
1532 close(TARGET_FILE);
1533}
1534
1535
1536# ==========================================================================================
1537# GREENSTONE XML FUNCTIONS
1538
1539sub build_key_to_line_mapping_for_greenstone_xml
1540{
1541 my (@file_lines) = @_;
1542
1543 my %chunk_key_to_line_mapping = ();
1544 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1545 my $line = $file_lines[$i];
1546 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1547
1548 # Line contains a string to translate
1549 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1550 my $chunk_key = $1;
1551 $line =~ s/\s*$//; # Remove any nasty whitespace
1552 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1553
1554 # While there is still text of the string to go...
1555 my $startline = $i;
1556 while ($line !~ /<\/Text>$/) {
1557 $i++;
1558 if ($i == scalar(@file_lines)) {
1559 &throw_fatal_error("Could not find end of string $chunk_key.");
1560 }
1561 $line = $file_lines[$i];
1562 $line =~ s/\s*$//; # Remove any nasty whitespace
1563 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1564 }
1565
1566 # Map from chunk key to line
1567 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1568 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1569 }
1570 else {
1571 &throw_fatal_error("Duplicate key $chunk_key.");
1572 }
1573 }
1574 }
1575
1576 return %chunk_key_to_line_mapping;
1577}
1578
1579
1580sub import_chunk_from_greenstone_xml
1581{
1582 my ($chunk_text) = @_;
1583
1584 # Simple: just remove the Text tags
1585 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1586 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1587 $chunk_text =~ s/<\/Text>$//;
1588
1589 return $chunk_text;
1590}
1591
1592
1593sub get_greenstone_xml_chunk_gti_comment
1594{
1595 my ($chunk_text) = @_;
1596
1597 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1598 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1599 return $1;
1600 }
1601
1602 return undef;
1603}
1604
1605
1606sub is_greenstone_xml_chunk_automatically_translated
1607{
1608 # No greenstone XML chunks are automatically translated
1609 return 0;
1610}
1611
1612
1613sub write_translated_greenstone_xml
1614{
1615 my $source_file = shift(@_); # Not used
1616 my @source_file_lines = @{shift(@_)};
1617 my $source_file_key_to_text_mapping = shift(@_);
1618 my $target_file = shift(@_);
1619 my @target_file_lines = @{shift(@_)}; # Not used
1620 my $target_file_key_to_text_mapping = shift(@_);
1621 my $target_file_key_to_gti_comment_mapping = shift(@_);
1622 my $target_language_code = shift(@_); # Not used
1623
1624 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1625 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1626 my %source_file_line_to_key_mapping = ();
1627 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1628 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1629 }
1630
1631 # Write the new target file
1632 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1633 if (!open(TARGET_FILE, ">$target_file_path")) {
1634 &throw_fatal_error("Could not write target file $target_file_path.");
1635 }
1636
1637 # Model the new target file on the source file, with the target file translations
1638 my $source_file_line_number = 0;
1639 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1640 # Fill in the gaps before this chunk starts
1641 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1642 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1643 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1644 print TARGET_FILE $source_file_lines[$source_file_line_number];
1645 $source_file_line_number++;
1646 }
1647 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1648
1649 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1650 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1651 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1652 $target_file_chunk_text =~ s/(\n)*$//g;
1653
1654 # If no translation exists for this chunk, show this, and move on
1655 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1656 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1657 next;
1658 }
1659
1660 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1661 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1662 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1663 $chunk_gti_comment =~ s/^Updated //;
1664 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1665 }
1666 print TARGET_FILE "\n";
1667 }
1668
1669 # Fill in the end of the file
1670 while ($source_file_line_number < scalar(@source_file_lines)) {
1671 print TARGET_FILE $source_file_lines[$source_file_line_number];
1672 $source_file_line_number++;
1673 }
1674
1675 close(TARGET_FILE);
1676}
1677
1678
1679# ==========================================================================================
1680# GREENSTONE3 FUNCTIONS
1681
1682sub get_all_chunks_for_gs3
1683{
1684 # The code of the target language (ensure it is lowercase)
1685 my $target_language_code = lc(shift(@_));
1686 my $translation_file_key = lc(shift(@_));
1687
1688 # Check that the necessary arguments were supplied
1689 if (!$target_language_code) {
1690 &throw_fatal_error("Missing command argument.");
1691 }
1692
1693 # Get (and check) the translation configuration
1694 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1695
1696 my %source_files_key_to_text_mapping = ();
1697 my %target_files_key_to_text_mapping = ();
1698 my %source_files_key_to_last_update_date_mapping = ();
1699 my %target_files_key_to_last_update_date_mapping = ();
1700
1701 &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1702
1703 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1704 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1705
1706 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1707 return $xml_response;
1708}
1709
1710
1711sub get_first_n_chunks_requiring_work_for_gs3
1712{
1713 # The code of the target language (ensure it is lowercase)
1714 my $target_language_code = lc(shift(@_));
1715 # The key of the file to translate (ensure it is lowercase)
1716 my $translation_file_key = lc(shift(@_));
1717 # The number of chunks to return (defaults to one if not specified)
1718 my $num_chunks_to_return = shift(@_) || "1";
1719
1720 # Check that the necessary arguments were supplied
1721 if (!$target_language_code || !$translation_file_key) {
1722 &throw_fatal_error("Missing command argument.");
1723 }
1724
1725 my %source_files_key_to_text_mapping = ();
1726 my %target_files_key_to_text_mapping = ();
1727 my %source_files_key_to_last_update_date_mapping = ();
1728 my %target_files_key_to_last_update_date_mapping = ();
1729
1730 &build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1731 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1732
1733 # Determine the target file chunks requiring translation
1734 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1735 # Determine the target file chunks requiring updating
1736 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1737 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1738 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1739
1740 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
1741 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1742 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1743 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1744
1745 return $xml_response;
1746}
1747
1748
1749
1750sub build_gs3_configuration
1751{
1752 my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
1753 $source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
1754
1755 my $source_file_directory = "greenstone3";
1756 my $translation_file_type = "resource_bundle";
1757
1758 foreach my $interface_file_key (@gs3_interface_files) {
1759
1760 &log_message("Greenstone 3 interface file: " . $interface_file_key);
1761
1762 # Parse the source language and target language files
1763 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
1764 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1765 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
1766 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1767 my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
1768
1769 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
1770 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
1771 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
1772 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1773 my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
1774
1775
1776 # Filter out any automatically translated chunks
1777 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1778 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
1779 delete $source_file_key_to_line_mapping{$chunk_key};
1780 delete $target_file_key_to_line_mapping{$chunk_key};
1781 }
1782 }
1783
1784 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
1785 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
1786
1787 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
1788 my $global_chunk_key = "$interface_file_key.$chunk_key";
1789 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
1790 $source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
1791
1792 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
1793 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
1794 $target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
1795 }
1796 }
1797 }
1798}
1799
1800
1801sub write_translated_gs3interface
1802{
1803 my $source_file_key_to_text_mapping = shift(@_);
1804 my $target_file_key_to_text_mapping = shift(@_);
1805 my $target_file_key_to_gti_comment_mapping = shift(@_);
1806 my $target_language_code = shift(@_);
1807
1808 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1809
1810 my %translated_interface_file_keys = ();
1811 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
1812 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
1813 if (!defined $translated_interface_file_keys{$1}) {
1814 &log_message("Updated interface file: " . $1);
1815 $translated_interface_file_keys{$1}="";
1816 }
1817 }
1818 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
1819
1820 my $source_file_directory = "greenstone3";
1821
1822 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
1823
1824 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1825 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
1826 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
1827 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1828 my %source_file_line_to_key_mapping = ();
1829 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1830 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1831 }
1832
1833 # Write the new target file
1834 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
1835 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1836 if (!open(TARGET_FILE, ">$target_file_path")) {
1837 &throw_fatal_error("Could not write target file $target_file_path.");
1838 }
1839
1840 # Model the new target file on the source file, with the target file translations
1841 my $source_file_line_number = 0;
1842 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1843 # Fill in the gaps before this chunk starts
1844 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1845 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1846 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1847 print TARGET_FILE $source_file_lines[$source_file_line_number];
1848 $source_file_line_number++;
1849 }
1850 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1851
1852 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1853 my $global_chunk_key = "$interface_file_key.$chunk_key";
1854 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
1855 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
1856
1857 # If no translation exists for this chunk, show this, and move on
1858 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1859 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1860 next;
1861 }
1862
1863 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1864 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
1865 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
1866 }
1867 print TARGET_FILE "\n";
1868 }
1869
1870 close(TARGET_FILE);
1871 }
1872}
1873
1874&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.