source: main/trunk/greenstone2/bin/script/gti.pl@ 30719

Last change on this file since 30719 was 30719, checked in by ak19, 5 years ago

Redoing the work for commits 30681 and 30687, to handle escaped colons (or equal separators) in chunk_keys of properties files used by GTI. Kathy recently introduced escaped colons into the metadata_names.properties files, but the GTI code did not yet support this.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 95.8 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en"; # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48 'file_type' => "macrofile",
49 'source_file' => "macros/english.dm",
50 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53 'file_type' => "macrofile",
54 'source_file' => "macros/english2.dm",
55 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58# 'file_type' => "macrofile",
59# 'source_file' => "macros/paperspast-english.dm",
60# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64 'file_type' => "resource_bundle",
65 'source_file' => "gli/classes/dictionary.properties",
66 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70 'file_type' => "greenstone_xml",
71 'source_file' => "gli/help/en/help.xml",
72 'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76 'file_type' => "resource_bundle",
77 'source_file' => "perllib/strings.properties",
78 'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82 'file_type' => "resource_bundle",
83 'source_file' => "gsinstaller/LanguagePack.properties",
84 'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94 'file_type' => "resource_bundle",
95 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96 'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102 'file_type' => "resource_bundle",
103 'source_file' => "greenstone3",
104 'target_file' => "greenstone3"
105},
106
107# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
108# from http://svn.greenstone.org/main/trunk/gs3-collection-configs
109{ 'key' => "gs3colcfg",
110 'file_type' => "resource_bundle",
111 'source_file' => "gs3-collection-configs",
112 'target_file' => "gs3-collection-configs"
113}
114];
115
116my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo");
117
118my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
119#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
120
121# Auxilliary GS3 interface files. This list is not used at present
122# Combine with above list if generating translation spreadsheet for all interface files
123my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
124
125my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
126
127# Not: i18n, log4j
128
129sub main
130{
131 # Get the command to process, and any arguments
132 my $gti_command = shift(@_);
133 my @gti_command_arguments = @_;
134 my $module = $_[1];
135
136 # for GS3, set gsdl_root_dir to GSDL3HOME
137 #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
138 #if($ENV{'GSDL3SRCHOME'}) {
139 # $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
140 # $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
141 #}
142 #}
143
144 # Open the GTI log file for appending, or write to STDERR if that fails
145 if (!open(GTI_LOG, ">>$gti_log_file")) {
146 open(GTI_LOG, ">&STDERR");
147 }
148
149 # Log the command that launched this script
150 &log_message("Command: $0 @ARGV");
151
152 # Check that a command was supplied
153 if (!$gti_command) {
154 &throw_fatal_error("Missing command.");
155 }
156
157 # Process the command
158 if ($gti_command =~ /^get-all-chunks$/i) {
159 # Check that GS3 interface is the target
160 if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg
161 print &get_all_chunks_for_gs3(@gti_command_arguments);
162 } else {
163 print &get_all_chunks(@gti_command_arguments);
164 }
165 }
166 elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
167 if ($module =~ m/^gs3/) {
168 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
169 } else {
170 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
171 }
172 }
173 elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
174 if ($module =~ m/^gs3/) {
175 print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
176 } else {
177 print &get_uptodate_chunks(@gti_command_arguments);
178 }
179 }
180 elsif ($gti_command =~ /^get-language-status$/i) {
181 print &get_language_status(@gti_command_arguments);
182 }
183 elsif ($gti_command =~ /^search-chunks$/i) {
184 print &search_chunks(@gti_command_arguments);
185 }
186 elsif ($gti_command =~ /^submit-translations$/i) {
187 # This command cannot produce any output since it reads input
188 &submit_translations(@gti_command_arguments);
189 }
190 elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
191 # This command cannot produce any output since it reads input
192 &create_glihelp_zip_file(@gti_command_arguments);
193 }
194 else {
195 # The command was not recognized
196 &throw_fatal_error("Unknown command \"$gti_command\".");
197 }
198}
199
200
201sub throw_fatal_error
202{
203 my $error_message = shift(@_);
204
205 # Write an XML error response
206 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
207 print "<GTIResponse>\n";
208 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
209 print "</GTIResponse>\n";
210
211 # Log the error message, then die
212 &log_message("Error: $error_message");
213 die "\n";
214}
215
216
217sub log_message
218{
219 my $log_message = shift(@_);
220 print GTI_LOG time() . " -- " . $log_message . "\n";
221}
222
223
224sub get_all_chunks
225{
226 # The code of the target language (ensure it is lowercase)
227 my $target_language_code = lc(shift(@_));
228 # The key of the file to translate (ensure it is lowercase)
229 my $translation_file_key = lc(shift(@_));
230
231 # Check that the necessary arguments were supplied
232 if (!$target_language_code || !$translation_file_key) {
233 &throw_fatal_error("Missing command argument.");
234 }
235
236 # Get (and check) the translation configuration
237 my ($source_file, $target_file, $translation_file_type)
238 = &get_translation_configuration($target_language_code, $translation_file_key);
239
240 # Parse the source language and target language files
241 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
242 my @source_file_lines = &read_file_lines($source_file_path);
243 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
244
245 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
246 my @target_file_lines = &read_file_lines($target_file_path);
247 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
248
249 # Filter out any automatically translated chunks
250 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
251 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
252 delete $source_file_key_to_line_mapping{$chunk_key};
253 delete $target_file_key_to_line_mapping{$chunk_key};
254 }
255 }
256
257 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
258 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
259 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
260 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
261
262 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
263 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
264
265 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
266
267 return $xml_response;
268}
269
270
271sub get_uptodate_chunks
272{
273 # The code of the target language (ensure it is lowercase)
274 my $target_language_code = lc(shift(@_));
275 # The key of the file to translate (ensure it is lowercase)
276 my $translation_file_key = lc(shift(@_));
277
278 # Check that the necessary arguments were supplied
279 if (!$target_language_code || !$translation_file_key) {
280 &throw_fatal_error("Missing command argument.");
281 }
282
283 # Get (and check) the translation configuration
284 my ($source_file, $target_file, $translation_file_type)
285 = &get_translation_configuration($target_language_code, $translation_file_key);
286
287 # Parse the source language and target language files
288 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
289 my @source_file_lines = &read_file_lines($source_file_path);
290 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
291
292 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
293 my @target_file_lines = &read_file_lines($target_file_path);
294 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
295
296 # Filter out any automatically translated chunks
297 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
298 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
299 delete $source_file_key_to_line_mapping{$chunk_key};
300 delete $target_file_key_to_line_mapping{$chunk_key};
301 }
302 }
303
304 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
305 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
306 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
307 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
308
309 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
310 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
311
312
313 # Chunks needing updating are those in the target file that have been more recently edited in the source file
314 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
315 my @uptodate_target_file_keys = ();
316 foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
317 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
318 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
319
320 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
321
322 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
323 # &log_message("Chunk with key $chunk_key needs updating.");
324 push(@uptodate_target_file_keys, $chunk_key);
325 }
326 }
327
328 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
329
330 return $xml_response;
331}
332
333
334sub get_first_n_chunks_requiring_work
335{
336 # The code of the target language (ensure it is lowercase)
337 my $target_language_code = lc(shift(@_));
338 # The key of the file to translate (ensure it is lowercase)
339 my $translation_file_key = lc(shift(@_));
340 # The number of chunks to return (defaults to one if not specified)
341 my $num_chunks_to_return = shift(@_) || "1";
342
343 # Check that the necessary arguments were supplied
344 if (!$target_language_code || !$translation_file_key) {
345 &throw_fatal_error("Missing command argument.");
346 }
347
348 # Get (and check) the translation configuration
349 my ($source_file, $target_file, $translation_file_type)
350 = &get_translation_configuration($target_language_code, $translation_file_key);
351
352 # Parse the source language and target language files
353 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
354 my @source_file_lines = &read_file_lines($source_file_path);
355 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
356
357 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
358 my @target_file_lines = &read_file_lines($target_file_path);
359 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
360
361 # Filter out any automatically translated chunks
362 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
363 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
364 delete $source_file_key_to_line_mapping{$chunk_key};
365 delete $target_file_key_to_line_mapping{$chunk_key};
366 }
367 }
368
369 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
372 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
373
374 # Determine the target file chunks requiring translation
375 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
376 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
377
378 # Determine the target file chunks requiring updating
379 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
380 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
381 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
382 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
383
384 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
385
386 return $xml_response;
387}
388
389
390sub get_language_status
391{
392 # The code of the target language (ensure it is lowercase)
393 my $target_language_code = lc(shift(@_));
394
395 # Check that the necessary arguments were supplied
396 if (!$target_language_code) {
397 &throw_fatal_error("Missing command argument.");
398 }
399
400 # Form an XML response to the command
401 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
402 $xml_response .= "<GTIResponse>\n";
403 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
404
405 foreach my $translation_file (@$gti_translation_files) {
406 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
407 my $target_file_name = "";
408
409 if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg
410 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
411 &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
412
413 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
414 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
415
416 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
417 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
418 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
419 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
420 }
421 else {
422 # Get (and check) the translation configuration
423 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
424 $target_file_name = $target_file;
425
426 # Parse the source language and target language files
427 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
428 my @source_file_lines = &read_file_lines($source_file_path);
429 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
430
431 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
432 my @target_file_lines = &read_file_lines($target_file_path);
433 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
434
435 # Filter out any automatically translated chunks
436 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
437 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
438 delete $source_file_key_to_line_mapping{$chunk_key};
439 delete $target_file_key_to_line_mapping{$chunk_key};
440 }
441 }
442
443 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
444 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
445
446 # Determine the target file chunks requiring translation
447 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
448
449 # Determine the target file chunks requiring updating
450 my @target_file_keys_requiring_updating = ();
451 if (-e $target_file_path) {
452 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
453 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
454 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
455 }
456
457 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
458 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
459 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
460 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
461 }
462
463 &log_message("Status of " . $translation_file->{'key'});
464 &log_message("Number of source chunks: " . $num_source_chunks);
465 &log_message("Number of target chunks: " . $num_target_chunks);
466 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
467 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
468
469 $xml_response .= " <TranslationFile"
470 . " key=\"" . $translation_file->{'key'} . "\""
471 . " target_file_path=\"" . $target_file_name . "\""
472 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
473 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
474 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
475 }
476
477 $xml_response .= " </LanguageStatus>\n";
478
479 $xml_response .= "</GTIResponse>\n";
480 return $xml_response;
481}
482
483
484sub search_chunks
485{
486 # The code of the target language (ensure it is lowercase)
487 my $target_language_code = lc(shift(@_));
488 # The key of the file to translate (ensure it is lowercase)
489 my $translation_file_key = lc(shift(@_));
490 # The query string
491 my $query_string = join(' ', @_);
492
493 # Check that the necessary arguments were supplied
494 if (!$target_language_code || !$translation_file_key || !$query_string) {
495 &throw_fatal_error("Missing command argument.");
496 }
497
498 my ($source_file, $target_file, $translation_file_type) = ();
499 my %source_file_key_to_text_mapping = ();
500 my %target_file_key_to_text_mapping = ();
501
502
503 if ($translation_file_key !~ m/^gs3/) {
504 # Get (and check) the translation configuration
505 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
506
507 # Parse the source language and target language files
508 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
509 my @source_file_lines = &read_file_lines($source_file_path);
510 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
511
512 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
513 my @target_file_lines = &read_file_lines($target_file_path);
514 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
515
516 # Filter out any automatically translated chunks
517 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
518 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
519 delete $source_file_key_to_line_mapping{$chunk_key};
520 delete $target_file_key_to_line_mapping{$chunk_key};
521 }
522 }
523
524 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
525 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
526 }
527 else {
528 # Not needed in this case
529 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
530 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
531 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
532 }
533
534 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
535 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
536
537 # Determine the target file chunks matching the query
538 my @target_file_keys_matching_query = ();
539 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
540 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
541 if ($target_file_text =~ /$query_string/i) {
542 # &log_message("Chunk with key $chunk_key matches query.");
543 push(@target_file_keys_matching_query, $chunk_key);
544 }
545 }
546
547 # Form an XML response to the command
548 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
549 $xml_response .= "<GTIResponse>\n";
550
551 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
552 foreach my $chunk_key (@target_file_keys_matching_query) {
553 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
554
555 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
556 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
557 $xml_response .= " </Chunk>\n";
558 }
559 $xml_response .= " </ChunksMatchingQuery>\n";
560
561 $xml_response .= "</GTIResponse>\n";
562 return $xml_response;
563}
564
565
566sub submit_translations
567{
568 # The code of the target language (ensure it is lowercase)
569 my $target_language_code = lc(shift(@_));
570 # The key of the file to translate (ensure it is lowercase)
571 my $translation_file_key = lc(shift(@_));
572 # The username of the translation submitter
573 my $submitter_username = shift(@_);
574 # Whether to submit a target chunk even if it hasn't changed
575 my $force_submission_flag = shift(@_);
576
577 # Check that the necessary arguments were supplied
578 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
579 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
580 die "\n";
581 }
582
583 my %source_file_key_to_text_mapping = ();
584 my %source_file_key_to_gti_comment_mapping = ();
585 my %target_file_key_to_text_mapping = ();
586 my %target_file_key_to_gti_comment_mapping = ();
587
588 my (@source_file_lines, @target_file_lines) = ();
589 my ($source_file, $target_file, $translation_file_type);
590
591
592 if ($translation_file_key !~ m/^gs3/) {
593 # Get (and check) the translation configuration
594 ($source_file, $target_file, $translation_file_type)
595 = &get_translation_configuration($target_language_code, $translation_file_key);
596
597 # Parse the source language and target language files
598 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
599 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
600 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
601 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
602
603 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
604 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
605 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
606 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
607 }
608 else {
609 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
610 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
611 }
612 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
613 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
614
615 # Submission date
616 my $day = (localtime)[3];
617 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
618 my $year = (localtime)[5] + 1900;
619 my $submission_date = "$day-$month-$year";
620
621 open(SUBMISSION, "-");
622 my @submission_lines = <SUBMISSION>;
623 close(SUBMISSION);
624
625 # Remove any nasty carriage returns
626 # &log_message("Submission:");
627 foreach my $submission_line (@submission_lines) {
628 $submission_line =~ s/\r$//;
629 #&log_message(" $submission_line");
630 }
631
632 my %source_file_key_to_submission_mapping = ();
633 my %target_file_key_to_submission_mapping = ();
634 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
635 # Read source file part of submission
636 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
637 my $chunk_key = $1;
638
639 # Read the source file text
640 my $source_file_chunk_text = "";
641 $i++;
642 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
643 $source_file_chunk_text .= $submission_lines[$i];
644 $i++;
645 }
646 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
647 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
648
649 #&log_message("Source file key: $chunk_key");
650 #&log_message("Source file text: $source_file_chunk_text");
651 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
652 }
653
654 # Read target file part of submission
655 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
656 my $chunk_key = $1;
657
658 # Read the target file text
659 my $target_file_chunk_text = "";
660 $i++;
661 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
662 $target_file_chunk_text .= $submission_lines[$i];
663 $i++;
664 }
665 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
666 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
667
668 #&log_message("Target file key: $chunk_key");
669 #&log_message("Target file text: $target_file_chunk_text");
670 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
671 }
672 }
673
674 # -----------------------------------------
675 # Validate the translation submissions
676 # -----------------------------------------
677
678 # Check that the translations are valid
679 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
680
681 # Kathy introduced escaped colons ("\:") into chunk keys in properties files (greenstone3/metadata_names),
682 # but they're not escaped in the submitted XML versions, nor are they escaped in memory (in the $chunk_key)
683
684 # Make sure the submitted chunk still exists in the source file
685 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
686 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
687 delete $source_file_key_to_submission_mapping{$chunk_key};
688 delete $target_file_key_to_submission_mapping{$chunk_key};
689 next;
690 }
691
692 # Make sure the submitted source chunk matches the source file chunk
693 if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
694 #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
695 #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
696 #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
697
698 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
699 &log_message("Submission source: |$source_file_key_to_submission_mapping{$chunk_key}|");
700 &log_message(" Source text: |$source_file_key_to_text_mapping{$chunk_key}|");
701 delete $source_file_key_to_submission_mapping{$chunk_key};
702 delete $target_file_key_to_submission_mapping{$chunk_key};
703 next;
704 }
705 }
706
707 # Apply the submitted translations
708 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
709 # Only apply the submission if it is a change, unless -force_submission has been specified
710 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
711 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
712 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
713 }
714 }
715
716 if ($translation_file_key !~ m/^gs3/) {
717 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
718 } else {
719 eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
720 }
721}
722
723
724sub create_glihelp_zip_file
725{
726 my $target_language_code = shift(@_);
727 my $translation_file_key = "glihelp";
728
729 &log_message("Creating GLI Help zip file for $target_language_code");
730
731 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
732
733 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
734 my $oldclasspath = $classpath;
735 if ( ! -e $classpath) {
736 $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
737 }
738 if ( ! -e $classpath) {
739 &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
740 }
741
742
743 my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
744 my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
745 my $os = $^O;
746 my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
747 my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
748 $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
749
750 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
751 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
752
753 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
754 if ( ! -e $gen_many_html_xsl_filepath) {
755 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
756 }
757
758 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
759 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
760
761 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
762 $target_file_directory = $target_file_directory."/";
763
764 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
765
766 # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
767 my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
768 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
769
770 # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
771 my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
772 my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
773 &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
774
775 # In gli/help/<newlang>/help.xml, replace all occurrences of
776 # <Text id="1">This text in en will be removed for new langcode</Text>
777 # with <!-- Missing translation: 1 -->
778 open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
779 my $help_xml_contents;
780 # Read in the entire contents of the file in one hit
781 sysread(FIN, $help_xml_contents, -s FIN);
782 close(FIN);
783
784 $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
785
786 open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
787 print FOUT $help_xml_contents;
788 close(FOUT);
789 }
790
791 my $perl_exec = &util::get_perl_exec();
792 my $java_exec = "java";
793 if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
794 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
795 }
796
797 #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
798 my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
799 #&throw_fatal_error("RAN gti command: $cmd");
800 my $response = `$cmd`;
801
802 #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
803 $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
804 $response = `$cmd`;
805
806 # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
807 my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
808 if(!&FileUtils::directoryExists($tmpdir)) {
809 &FileUtils::makeDirectory($tmpdir);
810 }
811 #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
812 my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
813 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
814
815 $response = `$cmd`;
816}
817
818
819sub get_translation_configuration
820{
821 # Get the code of the target language
822 my $target_language_code = shift(@_);
823 # Get the key of the file to translate
824 my $translation_file_key = shift(@_);
825
826 # Read the translation data from the gti.cfg file
827 my ($source_file, $target_file, $translation_file_type) =
828 &get_translation_data_for($target_language_code, $translation_file_key);
829
830 # Check that the file to translate is defined in the gti.cfg file
831 if (!$source_file || !$target_file || !$translation_file_type) {
832 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
833 }
834
835 # Check that the source file exists
836 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
837 if (!-e $source_file_path) {
838 &throw_fatal_error("Source file $source_file_path does not exist.");
839 }
840
841 # Check that the source file is up to date
842 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
843 # unless ($translation_file_is_not_in_cvs) {
844 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
845 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
846 if ($source_file_cvs_status =~ /^C /) {
847 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
848 }
849 if ($source_file_cvs_status =~ /^M /) {
850 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
851 }
852 # }
853
854 return ($source_file, $target_file, $translation_file_type);
855}
856
857
858sub get_translation_data_for
859{
860 my ($target_language_code, $translation_file_key) = @_;
861
862 foreach my $translation_file (@$gti_translation_files) {
863 # If this isn't the correct translation file, move onto the next one
864 next if ($translation_file_key ne $translation_file->{'key'});
865
866 # Resolve the target language file
867 my $target_language_file = $translation_file->{'target_file'};
868 if ($target_language_file =~ /(\{.+\;.+\})/) {
869 my $unresolved_target_language_file_part = $1;
870
871 # Check for a special case for the target language code
872 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
873 my $resolved_target_language_file_part = $2;
874 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
875 }
876 # Otherwise use the last part as the default value
877 else {
878 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
879 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
880 }
881 }
882
883 # Resolve instances of {iso_639_1_target_language_name}
884 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
885 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
886 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
887
888 # Resolve instances of {target_language_code}
889 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
890
891 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
892}
893
894return ();
895}
896
897
898sub read_file_lines
899{
900 my ($file_path) = @_;
901
902 if (!open(FILE_IN, "<$file_path")) {
903 &log_message("Note: Could not open file $file_path.");
904 return ();
905 }
906 my @file_lines = <FILE_IN>;
907 close(FILE_IN);
908
909 return @file_lines;
910}
911
912
913sub build_key_to_line_mapping
914{
915 my ($file_lines, $translation_file_type) = @_;
916 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
917}
918
919
920sub build_key_to_text_mapping
921{
922 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
923
924 my %key_to_text_mapping = ();
925 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
926 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
927 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
928
929 my $chunk_text = @$file_lines[$chunk_starting_line];
930 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
931 $chunk_text .= @$file_lines[$l];
932 }
933
934 # Map from chunk key to text
935 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
936
937 #if($chunk_key =~ m/document\\/) {
938 #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
939 #&log_message("@@@ chunk key: $chunk_key");
940 #}
941
942 }
943
944 return %key_to_text_mapping;
945}
946
947
948sub build_key_to_last_update_date_mapping
949{
950 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
951
952 # If the files aren't in CVS then we can't tell anything about what needs updating
953 # return () if ($translation_file_is_not_in_cvs);
954
955 # Build a mapping from key to CVS date
956 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
957 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
958
959 # Build a mapping from key to comment date
960 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
961
962 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
963 my %key_to_last_update_date_mapping = ();
964 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
965 # Use the CVS date as a starting point
966 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
967 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
968
969 # If a comment date exists and it is after the CVS date, use that instead
970 # need to convert the comment date format to SVN format
971 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
972 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
973 my $chunk_comment_date = $1;
974 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
975 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
976 }
977 }
978 }
979
980 return %key_to_last_update_date_mapping;
981}
982
983
984sub build_key_to_cvs_date_mapping
985{
986 my ($filename, $translation_file_type) = @_;
987
988 # Use SVN to annotate each line of the file with the date it was last edited
989 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
990 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
991
992 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
993
994 my @cvs_annotated_file_lines_date = ();
995 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
996 # Extract the date from the SVN annotation at the front
997 # svn format : 2007-07-16
998 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
999
1000 push(@cvs_annotated_file_lines_date, $1);
1001
1002 # trim extra date information in svn annotation format
1003 # 15:42:49 +1200 (Wed, 21 Jun 2006)
1004 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1005 }
1006
1007 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1008 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1009
1010 my %key_to_cvs_date_mapping = ();
1011 foreach my $chunk_key (keys(%key_to_line_mapping)) {
1012 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1013 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1014
1015 # Find the date this chunk was last edited, from the CVS annotation
1016 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
1017 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1018 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1019 # This part of the chunk has been updated more recently
1020 $chunk_date = $cvs_annotated_file_lines_date[$l];
1021
1022 }
1023 }
1024
1025 # Map from chunk key to CVS date
1026 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1027 }
1028
1029 return %key_to_cvs_date_mapping;
1030}
1031
1032
1033sub build_key_to_gti_comment_mapping
1034{
1035 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1036
1037 my %key_to_gti_comment_mapping = ();
1038 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1039 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1040 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1041
1042 my $chunk_text = @$file_lines[$chunk_starting_line];
1043 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1044 $chunk_text .= @$file_lines[$l];
1045 }
1046
1047 # Map from chunk key to GTI comment
1048 my $chunk_gti_comment;
1049 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1050 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1051 }
1052
1053 return %key_to_gti_comment_mapping;
1054}
1055
1056
1057sub determine_chunks_requiring_translation
1058{
1059 my $source_file_key_to_text_mapping = shift(@_);
1060 my $target_file_key_to_text_mapping = shift(@_);
1061
1062 # Chunks needing translation are those in the source file with no translation in the target file
1063 my @target_file_keys_requiring_translation = ();
1064 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1065 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1066 # &log_message("Chunk with key $chunk_key needs translating.");
1067 push(@target_file_keys_requiring_translation, $chunk_key);
1068 }
1069 }
1070
1071 return @target_file_keys_requiring_translation;
1072}
1073
1074
1075sub determine_chunks_requiring_updating
1076{
1077 my $source_file_key_to_last_update_date_mapping = shift(@_);
1078 my $target_file_key_to_last_update_date_mapping = shift(@_);
1079
1080 # Chunks needing updating are those in the target file that have been more recently edited in the source file
1081 my @target_file_keys_requiring_updating = ();
1082 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1083 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1084 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1085
1086 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1087
1088 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1089 # &log_message("Chunk with key $chunk_key needs updating.");
1090 # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1091 push(@target_file_keys_requiring_updating, $chunk_key);
1092 }
1093 }
1094
1095 return @target_file_keys_requiring_updating;
1096}
1097
1098
1099sub is_chunk_automatically_translated
1100{
1101 my ($chunk_key, $translation_file_type) = @_;
1102 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1103}
1104
1105
1106sub make_text_xml_safe
1107{
1108 my $text = shift(@_);
1109 $text =~ s/\&/\&amp\;/g;
1110 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1111 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1112 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1113 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1114 $text =~ s/</\&lt\;/g;
1115 $text =~ s/>/\&gt\;/g;
1116 return $text;
1117}
1118
1119
1120sub unmake_text_xml_safe
1121{
1122 my $text = shift(@_);
1123 $text =~ s/\&lt\;/</g;
1124 $text =~ s/\&gt\;/>/g;
1125 $text =~ s/\&amp\;/\&/g;
1126 return $text;
1127}
1128
1129
1130# Returns 1 if $date1 is after $date2, 0 otherwise
1131sub is_date_after_cvs
1132{
1133 my ($date1, $date2) = @_;
1134 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
1135 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1136
1137 if(!defined $date1) {
1138 return 1;
1139 }
1140
1141 my @date1parts = split(/-/, $date1);
1142 my @date2parts = split(/-/, $date2);
1143
1144 # Compare year - nasty because we have rolled over into a new century
1145 my $year1 = $date1parts[2];
1146 if ($year1 < 80) {
1147 $year1 += 2000;
1148 }
1149 my $year2 = $date2parts[2];
1150 if ($year2 < 80) {
1151 $year2 += 2000;
1152 }
1153
1154 # Compare year
1155 if ($year1 > $year2) {
1156 return 1;
1157 }
1158 elsif ($year1 == $year2) {
1159 # Year is the same, so compare month
1160 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1161 return 1;
1162 }
1163 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1164 # Month is the same, so compare day
1165 if ($date1parts[0] > $date2parts[0]) {
1166 return 1;
1167 }
1168 }
1169 }
1170
1171 return 0;
1172}
1173
1174sub is_date_after
1175{
1176 my ($date1, $date2) = @_;
1177
1178 if(!defined $date1) {
1179 return 1;
1180 }
1181 if(!defined $date2) {
1182 return 0;
1183 }
1184
1185 # 16-Aug-2006
1186 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1187 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1188 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1189 $date1=$3 . "-" . $months{$2} . "-" . $1;
1190 # print "** converted date1: $date1\n";
1191 }
1192 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1193 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1194 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1195 $date2=$3 . "-" . $months{$2} . "-" . $1;
1196 # print "** converted date2: $date2\n";
1197 }
1198
1199
1200 # 2006-08-16
1201 my @date1parts = split(/-/, $date1);
1202 my @date2parts = split(/-/, $date2);
1203
1204 # Compare year
1205 if ($date1parts[0] > $date2parts[0]) {
1206 return 1;
1207 }
1208 elsif ($date1parts[0] == $date2parts[0]) {
1209 # Year is the same, so compare month
1210 if ($date1parts[1] > $date2parts[1]) {
1211 return 1;
1212 }
1213 elsif ($date1parts[1] == $date2parts[1]) {
1214 # Month is the same, so compare day
1215 if ($date1parts[2] > $date2parts[2]) {
1216 return 1;
1217 }
1218 }
1219 }
1220
1221 return 0;
1222}
1223
1224
1225sub create_xml_response_for_chunks_requiring_work
1226{
1227 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1228
1229 # Form an XML response to the command
1230 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1231 $xml_response .= "<GTIResponse>\n";
1232 $xml_response .= " <TranslationFile"
1233 . " key=\"" . $translation_file_key . "\""
1234 . " target_file_path=\"" . $target_file . "\""
1235 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1236 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1237 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1238
1239 # Do chunks requiring translation first
1240 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1241 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1242 }
1243 else {
1244 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1245 }
1246
1247 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1248 foreach my $chunk_key (@sorted_chunk_keys) {
1249 last if ($num_chunks_to_return == 0);
1250
1251 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1252 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1253
1254 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1255 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1256 $xml_response .= " <TargetFileText></TargetFileText>\n";
1257 $xml_response .= " </Chunk>\n";
1258
1259 $num_chunks_to_return--;
1260 }
1261
1262 $xml_response .= " </ChunksRequiringTranslation>\n";
1263
1264 # Then do chunks requiring updating
1265 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1266 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1267 }
1268 else {
1269 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1270 }
1271
1272 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1273 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1274 foreach my $chunk_key (@sorted_chunk_keys) {
1275 last if ($num_chunks_to_return == 0);
1276
1277 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1278 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1279 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1280 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1281
1282 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1283 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1284 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1285 $xml_response .= " </Chunk>\n";
1286
1287 $num_chunks_to_return--;
1288 }
1289
1290 $xml_response .= " </ChunksRequiringUpdating>\n";
1291
1292 $xml_response .= "</GTIResponse>\n";
1293
1294 return $xml_response;
1295}
1296
1297sub create_xml_response_for_uptodate_chunks
1298{
1299 my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1300
1301 # Form an XML response to the command
1302 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1303 $xml_response .= "<GTIResponse>\n";
1304 $xml_response .= " <TranslationFile"
1305 . " key=\"" . $translation_file_key . "\""
1306 . " target_file_path=\"" . $target_file . "\""
1307 . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1308
1309
1310 # Then do chunks requiring updating
1311 $xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1312
1313
1314 # foreach my $chunk_key (@uptodate_target_file_keys) {
1315 my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1316 foreach my $chunk_key (@sorted_chunk_keys) {
1317
1318 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1319 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1320 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1321 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1322
1323 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1324 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1325 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1326 $xml_response .= " </Chunk>\n";
1327
1328 }
1329
1330 $xml_response .= " </UptodateChunks>\n";
1331
1332 $xml_response .= "</GTIResponse>\n";
1333
1334 return $xml_response;
1335}
1336
1337sub create_xml_response_for_all_chunks
1338{
1339 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1340
1341 # Form an XML response to the command
1342 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1343 $xml_response .= "<GTIResponse>\n";
1344 $xml_response .= " <TranslationFile"
1345 . " key=\"" . $translation_file_key . "\""
1346 . " target_file_path=\"" . $target_file . "\"\/>\n";
1347
1348 # Do all the chunks
1349 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1350
1351 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1352 foreach my $chunk_key (@sorted_chunk_keys) {
1353 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1354 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1355
1356 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1357 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1358 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1359 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1360 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1361 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1362 }
1363 else {
1364 $xml_response .= " <TargetFileText></TargetFileText>\n";
1365 }
1366
1367 $xml_response .= " </Chunk>\n";
1368 }
1369 $xml_response .= " </Chunks>\n";
1370
1371 $xml_response .= "</GTIResponse>\n";
1372 return $xml_response;
1373}
1374
1375
1376
1377# ==========================================================================================
1378# MACROFILE FUNCTIONS
1379
1380sub build_key_to_line_mapping_for_macrofile
1381{
1382 my (@file_lines) = @_;
1383
1384 my $macro_package;
1385 my %chunk_key_to_line_mapping = ();
1386 # Process the contents of the file, line by line
1387 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1388 my $line = $file_lines[$i];
1389 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1390
1391 # Check if a new package is being defined
1392 if ($line =~ m/^package\s+(.+)/) {
1393 $macro_package = $1;
1394 }
1395
1396 # Line contains a macro name
1397 elsif ($line =~ m/^(_\w+_)/) {
1398 my $macro_key = $1;
1399 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1400
1401 # While there is still text of the macro to go...
1402 my $startline = $i;
1403 while ($line !~ /\}$/) {
1404 $i++;
1405 if ($i == scalar(@file_lines)) {
1406 &throw_fatal_error("Could not find end of macro $macro_key.");
1407 }
1408 $line = $file_lines[$i];
1409 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1410 }
1411
1412 # The chunk key consists of the package name and the macro key
1413 my $chunk_key = $macro_package . "." . $macro_key;
1414 # Map from chunk key to line
1415 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1416 }
1417
1418 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1419 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1420 # The chunk key consists of package name and macro key
1421 my $chunk_key = $macro_package . "." . $1;
1422 # Map from chunk key to line
1423 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1424}
1425}
1426
1427return %chunk_key_to_line_mapping;
1428}
1429
1430
1431sub import_chunk_from_macrofile
1432{
1433 my ($chunk_text) = @_;
1434
1435 # Is this an icon macro??
1436 if ($chunk_text =~ /^\#\# (.*)/) {
1437 # Extract image macro text
1438 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1439 $chunk_text = $1;
1440
1441 # Remove enclosing quotes
1442 $chunk_text =~ s/^\"//;
1443 $chunk_text =~ s/\"$//;
1444 }
1445
1446 # No, so it must be a text macro
1447 else {
1448 # Remove macro key
1449 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1450
1451 # Remove language specifier
1452 $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1453
1454 # Remove braces enclosing text
1455 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1456 }
1457
1458 return $chunk_text;
1459}
1460
1461
1462sub get_macrofile_chunk_gti_comment
1463{
1464 my ($chunk_text) = @_;
1465
1466 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1467 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1468 return $1;
1469}
1470
1471return undef;
1472}
1473
1474
1475sub is_macrofile_chunk_automatically_translated
1476{
1477 my ($chunk_key) = @_;
1478
1479 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1480 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1481 return 1;
1482 }
1483
1484 return 0;
1485}
1486
1487
1488# Use the source file to generate a target file that is formatted the same
1489sub write_translated_macrofile
1490{
1491 my $source_file = shift(@_); # Not used
1492 my @source_file_lines = @{shift(@_)};
1493 my $source_file_key_to_text_mapping = shift(@_);
1494 my $target_file = shift(@_);
1495 my @target_file_lines = @{shift(@_)};
1496 my $target_file_key_to_text_mapping = shift(@_);
1497 my $target_file_key_to_gti_comment_mapping = shift(@_);
1498 my $target_language_code = shift(@_);
1499
1500 # Build a mapping from source file line to chunk key
1501 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1502 my %source_file_line_to_key_mapping = ();
1503 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1504 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1505 }
1506 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1507 my $source_file_line_number = 0;
1508
1509 # Build a mapping from target file line to chunk key
1510 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1511 my %target_file_line_to_key_mapping = ();
1512 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1513 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1514 }
1515 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1516
1517 # Write the new target file
1518 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1519 if (!open(TARGET_FILE, ">$target_file_path")) {
1520 &throw_fatal_error("Could not write target file $target_file_path.");
1521 }
1522
1523 # Use the header from the target file, to keep language and author information
1524 if (scalar(@target_file_line_keys) > 0) {
1525 my $target_file_line_number = 0;
1526 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1527 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1528 my $target_file_line = $target_file_lines[$target_file_line_number];
1529 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1530 print TARGET_FILE $target_file_line;
1531 $target_file_line_number++;
1532 }
1533
1534 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1535 }
1536
1537 # Model the new target file on the source file, with the target file translations
1538 foreach my $line_key (@source_file_line_keys) {
1539 # Fill in the gaps before this chunk starts
1540 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1541 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1542 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1543 print TARGET_FILE $source_file_lines[$source_file_line_number];
1544 $source_file_line_number++;
1545 }
1546 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1547
1548 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1549 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1550 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1551
1552 my $macrofile_key = $chunk_key;
1553 $macrofile_key =~ s/^(.+?)\.//;
1554
1555 # If no translation exists for this chunk, show this, and move on
1556 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1557 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1558 next;
1559 }
1560
1561 # Grab the source chunk text
1562 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1563 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1564 $source_file_chunk .= $source_file_lines[$l];
1565 }
1566
1567 # Is this an icon macro??
1568 if ($source_file_chunk =~ /^\#\# (.*)/) {
1569 # Escape any newline and question mark characters so the source text is replaced correctly
1570 $source_file_chunk_text =~ s/\\/\\\\/g;
1571 $source_file_chunk_text =~ s/\?/\\\?/g;
1572
1573 # Build the new target chunk from the source chunk
1574 my $target_file_chunk = $source_file_chunk;
1575 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1576 $target_file_chunk =~ s/(\s)*$//;
1577 print TARGET_FILE "$target_file_chunk";
1578 }
1579
1580 # No, it is just a normal text macro
1581 else {
1582 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1583 }
1584
1585 # Add the "updated" comment, if one exists
1586 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1587 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1588 }
1589 print TARGET_FILE "\n";
1590}
1591
1592close(TARGET_FILE);
1593}
1594
1595
1596sub sort_by_line
1597{
1598 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1599}
1600
1601
1602# ==========================================================================================
1603# RESOURCE BUNDLE FUNCTIONS
1604
1605# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1606sub build_key_to_line_mapping_for_resource_bundle
1607{
1608 my (@file_lines) = @_;
1609
1610 my %chunk_key_to_line_mapping = ();
1611
1612 my $chunk_key;
1613 my $startindex = -1;
1614
1615 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1616 my $line = $file_lines[$i];
1617 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1618
1619 # a property line has a colon/equals sign as separator that is NOT escaped with a backslash (both keys and values
1620 # can use the colon or = sign. But in the key, such a char is always escaped. Unfortunately, they've not always been
1621 # escaped in the values. So we get the left most occurrence by not doing a greedy match (use ? to not be greedy).
1622 # So find the first :/= char not preceded by \. That will be the true separator of a chunk_key and its value chunk_text
1623
1624 if ($line =~ m/^(\S*?[^\\])[:|=](.*)$/) { #if ($line =~ /^(\S+?)[:|=](.*)$/) {
1625 # Line contains a dictionary string
1626
1627 # Unused but useful: http://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
1628 # http://perldoc.perl.org/perlvar.html
1629
1630 $chunk_key = $1;
1631 # remove the escaping of any :/= property separator from the chunk_key in memory,
1632 # to make comparison with its unescaped version during submissions easier. Will write out with escaping.
1633 $chunk_key =~ s/\\([:=])/$1/g;
1634
1635 $startindex = $i;
1636 }
1637 if ($startindex != -1) {
1638 if($line !~ m/\\$/) { # line finished
1639 # $i keeps track of the line at which this property (chunk_key) finishes
1640
1641 # Map from chunk key to line
1642 $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1643 $startindex = -1;
1644 $chunk_key = "";
1645 }
1646 }
1647 }
1648
1649 return %chunk_key_to_line_mapping;
1650}
1651
1652
1653sub import_chunk_from_resource_bundle
1654{
1655 my ($chunk_text) = @_;
1656
1657 # Simple: just remove string key.
1658 # But key can contain an escaped separator (\: or \=).
1659 # So just as in the previous subroutine, find the first (leftmost) : or = char not preceded by \.
1660 # That will be the true separator of a chunk_key and its value chunk_text
1661 $chunk_text =~ s/^(\S*?[^\\])[:|=](\s*)//s; # $chunk_text =~ s/^(\S+)[:|=](\s*)//s;
1662
1663 $chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
1664 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1665
1666 return $chunk_text;
1667}
1668
1669
1670sub get_resource_bundle_chunk_gti_comment
1671{
1672 my ($chunk_text) = @_;
1673
1674 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1675 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1676 return $1;
1677 }
1678
1679 return undef;
1680}
1681
1682
1683sub is_resource_bundle_chunk_automatically_translated
1684{
1685 # No resource bundle chunks are automatically translated
1686 return 0;
1687}
1688
1689
1690sub write_translated_resource_bundle
1691{
1692 my $source_file = shift(@_); # Not used
1693 my @source_file_lines = @{shift(@_)};
1694 my $source_file_key_to_text_mapping = shift(@_);
1695 my $target_file = shift(@_);
1696 my @target_file_lines = @{shift(@_)}; # Not used
1697 my $target_file_key_to_text_mapping = shift(@_);
1698 my $target_file_key_to_gti_comment_mapping = shift(@_);
1699 my $target_language_code = shift(@_); # Not used
1700
1701 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1702 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1703 my %source_file_line_to_key_mapping = ();
1704 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1705 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1706 }
1707
1708 # Write the new target file
1709 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1710 if (!open(TARGET_FILE, ">$target_file_path")) {
1711 &throw_fatal_error("Could not write target file $target_file_path.");
1712 }
1713
1714 # Model the new target file on the source file, with the target file translations
1715 my $source_file_line_number = 0;
1716 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1717 # Fill in the gaps before this chunk starts
1718 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1719 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1720 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1721 print TARGET_FILE $source_file_lines[$source_file_line_number];
1722 $source_file_line_number++;
1723 }
1724 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1725
1726 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1727 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1728 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1729
1730 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
1731 # since the key-value separator in a property resource bundle file is : or =
1732 my $escaped_chunk_key = $chunk_key;
1733 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
1734
1735 # If no translation exists for this chunk, show this, and move on
1736 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1737 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
1738 next;
1739 }
1740
1741 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
1742 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1743 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1744 }
1745 print TARGET_FILE "\n";
1746 }
1747
1748 close(TARGET_FILE);
1749}
1750
1751
1752# ==========================================================================================
1753# GREENSTONE XML FUNCTIONS
1754
1755sub build_key_to_line_mapping_for_greenstone_xml
1756{
1757 my (@file_lines) = @_;
1758
1759 my %chunk_key_to_line_mapping = ();
1760 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1761 my $line = $file_lines[$i];
1762 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1763
1764 # Line contains a string to translate
1765 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1766 my $chunk_key = $1;
1767 $line =~ s/\s*$//; # Remove any nasty whitespace
1768 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1769
1770 # While there is still text of the string to go...
1771 my $startline = $i;
1772 while ($line !~ /<\/Text>$/) {
1773 $i++;
1774 if ($i == scalar(@file_lines)) {
1775 &throw_fatal_error("Could not find end of string $chunk_key.");
1776 }
1777 $line = $file_lines[$i];
1778 $line =~ s/\s*$//; # Remove any nasty whitespace
1779 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1780 }
1781
1782 # Map from chunk key to line
1783 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1784 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1785 }
1786 else {
1787 &throw_fatal_error("Duplicate key $chunk_key.");
1788 }
1789 }
1790 }
1791
1792 return %chunk_key_to_line_mapping;
1793}
1794
1795
1796sub import_chunk_from_greenstone_xml
1797{
1798 my ($chunk_text) = @_;
1799
1800 # Simple: just remove the Text tags
1801 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1802 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1803 $chunk_text =~ s/<\/Text>$//;
1804
1805 return $chunk_text;
1806}
1807
1808
1809sub get_greenstone_xml_chunk_gti_comment
1810{
1811 my ($chunk_text) = @_;
1812
1813 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1814 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1815 return $1;
1816 }
1817
1818 return undef;
1819}
1820
1821
1822sub is_greenstone_xml_chunk_automatically_translated
1823{
1824 # No greenstone XML chunks are automatically translated
1825 return 0;
1826}
1827
1828
1829sub write_translated_greenstone_xml
1830{
1831 my $source_file = shift(@_); # Not used
1832 my @source_file_lines = @{shift(@_)};
1833 my $source_file_key_to_text_mapping = shift(@_);
1834 my $target_file = shift(@_);
1835 my @target_file_lines = @{shift(@_)}; # Not used
1836 my $target_file_key_to_text_mapping = shift(@_);
1837 my $target_file_key_to_gti_comment_mapping = shift(@_);
1838 my $target_language_code = shift(@_); # Not used
1839
1840 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1841 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1842 my %source_file_line_to_key_mapping = ();
1843 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1844 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1845 }
1846
1847 # Write the new target file
1848 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1849 if (!open(TARGET_FILE, ">$target_file_path")) {
1850 &throw_fatal_error("Could not write target file $target_file_path.");
1851 }
1852
1853 # Model the new target file on the source file, with the target file translations
1854 my $source_file_line_number = 0;
1855 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1856 # Fill in the gaps before this chunk starts
1857 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1858 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1859 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1860 print TARGET_FILE $source_file_lines[$source_file_line_number];
1861 $source_file_line_number++;
1862 }
1863 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1864
1865 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1866 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1867 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1868 $target_file_chunk_text =~ s/(\n)*$//g;
1869
1870 # If no translation exists for this chunk, show this, and move on
1871 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1872 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1873 next;
1874 }
1875
1876 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1877 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1878 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1879 $chunk_gti_comment =~ s/^Updated //;
1880 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1881 }
1882 print TARGET_FILE "\n";
1883 }
1884
1885 # Fill in the end of the file
1886 while ($source_file_line_number < scalar(@source_file_lines)) {
1887 print TARGET_FILE $source_file_lines[$source_file_line_number];
1888 $source_file_line_number++;
1889 }
1890
1891 close(TARGET_FILE);
1892}
1893
1894
1895# ==========================================================================================
1896# GREENSTONE3 FUNCTIONS
1897
1898sub get_all_chunks_for_gs3
1899{
1900 # The code of the target language (ensure it is lowercase)
1901 my $target_language_code = lc(shift(@_));
1902 my $translation_file_key = lc(shift(@_));
1903
1904 # Check that the necessary arguments were supplied
1905 if (!$target_language_code) {
1906 &throw_fatal_error("Missing command argument.");
1907 }
1908
1909 # Get (and check) the translation configuration
1910 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1911
1912 my %source_files_key_to_text_mapping = ();
1913 my %target_files_key_to_text_mapping = ();
1914 my %source_files_key_to_last_update_date_mapping = ();
1915 my %target_files_key_to_last_update_date_mapping = ();
1916
1917 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1918
1919 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1920 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1921
1922 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1923 return $xml_response;
1924}
1925
1926
1927sub get_first_n_chunks_requiring_work_for_gs3
1928{
1929 # The code of the target language (ensure it is lowercase)
1930 my $target_language_code = lc(shift(@_));
1931 # The key of the file to translate (ensure it is lowercase)
1932 my $translation_file_key = lc(shift(@_));
1933 # The number of chunks to return (defaults to one if not specified)
1934 my $num_chunks_to_return = shift(@_) || "1";
1935
1936 # Check that the necessary arguments were supplied
1937 if (!$target_language_code || !$translation_file_key) {
1938 &throw_fatal_error("Missing command argument.");
1939 }
1940
1941 my %source_files_key_to_text_mapping = ();
1942 my %target_files_key_to_text_mapping = ();
1943 my %source_files_key_to_last_update_date_mapping = ();
1944 my %target_files_key_to_last_update_date_mapping = ();
1945
1946 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1947 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1948
1949 # Determine the target file chunks requiring translation
1950 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1951 # Determine the target file chunks requiring updating
1952 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1953 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1954 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1955
1956 my $download_target_filepath = "";
1957
1958
1959 # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1960
1961 # if there is no copy of the language files for download, there's also no link to the spreadsheet
1962 # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1963 # interface *.properties files,and link to that zip as the file for offline translation.
1964 # Selecting only properties files for English and the language they're working on (if the last exists)
1965
1966 # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1967 # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1968
1969 my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1970 my $tar_cmd = "tar -cvzf $zip";
1971
1972
1973 # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1974 my $curdir = `pwd`;
1975 chdir $gsdl_root_directory;
1976
1977 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
1978 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
1979
1980 # tar command will overwrite the previous version, but want to check we've created it
1981 if(&FileUtils::fileExists($zip)) {
1982 &FileUtils::removeFiles($zip);
1983 }
1984
1985 #my $tar_result = system($tar_cmd); # works but then interface breaks
1986 `$tar_cmd`;
1987 my $tar_result = $?;
1988
1989 if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1990 $download_target_filepath = $zip;
1991 } else {
1992 &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1993 }
1994
1995 # change back to original working directory (cgi-bin/linux probably)
1996 chdir $curdir;
1997
1998 # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
1999
2000
2001 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
2002 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
2003 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2004 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2005
2006 return $xml_response;
2007}
2008
2009# helper function
2010# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg)
2011# formatted correctly to go into a zip file
2012sub get_gs3_zip_file_listing
2013{
2014 my $target_language_code = shift(@_);
2015 my $sourcedir = shift(@_);
2016 my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
2017
2018 my $filelisting = "";
2019 foreach my $interface_file (@$files_array) {
2020
2021 my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
2022 my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2023
2024 $filelisting = "$filelisting $source_filepath";
2025 if(&FileUtils::fileExists($target_filepath)) {
2026 $filelisting = "$filelisting $target_filepath";
2027 }
2028 }
2029
2030 return $filelisting;
2031}
2032
2033sub get_uptodate_chunks_for_gs3
2034{
2035 # The code of the target language (ensure it is lowercase)
2036 my $target_language_code = lc(shift(@_));
2037 # The key of the file to translate (ensure it is lowercase)
2038 my $translation_file_key = lc(shift(@_));
2039 # The number of chunks to return (defaults to one if not specified)
2040 my $num_chunks_to_return = shift(@_) || "1";
2041
2042 # Check that the necessary arguments were supplied
2043 if (!$target_language_code || !$translation_file_key) {
2044 &throw_fatal_error("Missing command argument.");
2045 }
2046
2047 my %source_files_key_to_text_mapping = ();
2048 my %target_files_key_to_text_mapping = ();
2049 my %source_files_key_to_last_update_date_mapping = ();
2050 my %target_files_key_to_last_update_date_mapping = ();
2051
2052 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2053 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2054
2055
2056 # Chunks needing updating are those in the target file that have been more recently edited in the source file
2057 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2058 my @uptodate_target_file_keys = ();
2059 foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2060 my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2061 my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2062
2063 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
2064
2065 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2066 # &log_message("Chunk with key $chunk_key needs updating.");
2067 push(@uptodate_target_file_keys, $chunk_key);
2068 }
2069 }
2070
2071 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2072
2073 return $xml_response;
2074}
2075
2076
2077sub build_gs3_configuration
2078{
2079 my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2080 $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2081
2082 my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2083 my $files_array = \@gs3_interface_files;
2084
2085 if($translation_file_key eq "gs3colcfg") {
2086 $source_file_directory = "gs3-collection-configs";
2087 $files_array = \@gs3_col_cfg_files;
2088 }
2089 my $translation_file_type = "resource_bundle";
2090
2091 foreach my $interface_file_key (@$files_array) {
2092
2093 &log_message("Greenstone 3 interface file: " . $interface_file_key);
2094
2095 # Parse the source language and target language files
2096 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2097 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2098 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2099 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2100 #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2101
2102 my %source_file_key_to_gti_comment_or_last_updated_mapping;
2103 if($get_gti_comments_not_last_updated) {
2104 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2105 } else {
2106 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2107 }
2108
2109 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2110 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2111 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2112 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2113 #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2114
2115 my %target_file_key_to_gti_comment_or_last_updated_mapping;
2116 if($get_gti_comments_not_last_updated) {
2117 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2118 } else {
2119 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2120 }
2121
2122
2123 # Filter out any automatically translated chunks
2124 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2125 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2126 delete $source_file_key_to_line_mapping{$chunk_key};
2127 delete $target_file_key_to_line_mapping{$chunk_key};
2128 }
2129 }
2130
2131 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2132 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2133
2134 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2135 my $global_chunk_key = "$interface_file_key.$chunk_key";
2136 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2137 $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2138
2139 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2140 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2141 $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2142 }
2143 }
2144 }
2145}
2146
2147
2148sub write_translated_gs3interface
2149{
2150 my $translation_file_key = shift(@_);
2151 my $source_file_key_to_text_mapping = shift(@_);
2152 my $target_file_key_to_text_mapping = shift(@_);
2153 my $target_file_key_to_gti_comment_mapping = shift(@_);
2154 my $target_language_code = shift(@_);
2155
2156 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2157
2158 my %translated_interface_file_keys = ();
2159 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2160 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2161 if (!defined $translated_interface_file_keys{$1}) {
2162 &log_message("Updated interface file: " . $1);
2163 $translated_interface_file_keys{$1}="";
2164 }
2165 }
2166 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2167
2168 my $source_file_directory = "greenstone3";
2169 $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2170
2171 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2172
2173 # Build a mapping from chunk key to source file line, and from source file line to chunk key
2174 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2175 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2176 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2177 my %source_file_line_to_key_mapping = ();
2178 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2179 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2180 }
2181
2182 # Write the new target file
2183 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2184 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2185 if (!open(TARGET_FILE, ">$target_file_path")) {
2186 &throw_fatal_error("Could not write target file $target_file_path.");
2187 }
2188
2189 # Model the new target file on the source file, with the target file translations
2190 my $source_file_line_number = 0;
2191 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2192 # Fill in the gaps before this chunk starts
2193 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2194 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2195 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2196 print TARGET_FILE $source_file_lines[$source_file_line_number];
2197 $source_file_line_number++;
2198 }
2199 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2200
2201 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2202 my $global_chunk_key = "$interface_file_key.$chunk_key";
2203 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2204 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2205
2206 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
2207 # since the key-value separator in a property resource bundle file is : or =
2208 my $escaped_chunk_key = $chunk_key;
2209 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
2210
2211 # If no translation exists for this chunk, show this, and move on
2212 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2213 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
2214 next;
2215 }
2216
2217 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
2218 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2219 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2220 }
2221 print TARGET_FILE "\n";
2222 }
2223
2224 close(TARGET_FILE);
2225 }
2226}
2227
2228&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.