source: main/trunk/greenstone2/bin/script/gti.pl@ 36548

Last change on this file since 36548 was 36548, checked in by anupama, 20 months ago

gti.pl kept replacing = with : as key-value separator in the DEC properties files, where I really want it to stay as =, since I'd carefully escaped all : occurrences in the values already.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 98.7 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en"; # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48 'file_type' => "macrofile",
49 'source_file' => "macros/english.dm",
50 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;sr-bh-cyr:serbian-bh-cyr;sr-bh-lat:serbian-bh-lat;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53 'file_type' => "macrofile",
54 'source_file' => "macros/english2.dm",
55 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58# 'file_type' => "macrofile",
59# 'source_file' => "macros/paperspast-english.dm",
60# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64 'file_type' => "resource_bundle",
65 'source_file' => "gli/classes/dictionary.properties",
66 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70 'file_type' => "greenstone_xml",
71 'source_file' => "gli/help/en/help.xml",
72 'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76 'file_type' => "resource_bundle",
77 'source_file' => "perllib/strings.properties",
78 'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82 'file_type' => "resource_bundle",
83 'source_file' => "gsinstaller/LanguagePack.properties",
84 'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94 'file_type' => "resource_bundle",
95 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96 'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from https://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102 'file_type' => "resource_bundle",
103 'source_file' => "greenstone3",
104 'target_file' => "greenstone3"
105},
106
107# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
108# from https://svn.greenstone.org/main/trunk/gs3-collection-configs
109{ 'key' => "gs3colcfg",
110 'file_type' => "resource_bundle",
111 'source_file' => "gs3-collection-configs",
112 'target_file' => "gs3-collection-configs"
113},
114
115
116# collection config display items of GS3 port of Documented Examples Collections (dec). Check out as gs3-dec-col-cfgs
117# from https://svn.greenstone.org/main/trunk/gs3-dec-col-cfgs
118{ 'key' => "gs3deccolcfg",
119 'file_type' => "resource_bundle",
120 'source_file' => "gs3-dec-col-cfgs",
121 'target_file' => "gs3-dec-col-cfgs"
122}
123];
124
125my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo", "localsite");
126
127my @gs3_dec_col_cfg_files = ("authen-e", "bibtex-e", "dls-e", "garish-e", "gsarch-e", "image-e", "isis-e", "lomdemo-e", "manifest-demo-e", "marc-e", "oai-e", "pagedimg-e", "style-e", "wiki-e", "wrdpdf-e"); # "bibtex-supp-e"
128
129my @gs3_interface_files = ("interface_default", "core_servlet_dictionary", "metadata_names");
130#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
131
132# Auxilliary GS3 interface files. This list is not used at present
133# Combine with above list if generating translation spreadsheet for all interface files
134my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
135
136my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
137
138# Not: i18n, log4j
139
140sub main
141{
142 # Get the command to process, and any arguments
143 my $gti_command = shift(@_);
144 my @gti_command_arguments = @_;
145 my $module = $_[1];
146
147 # for GS3, set gsdl_root_dir to GSDL3HOME
148 #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
149 #if($ENV{'GSDL3SRCHOME'}) {
150 # $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
151 # $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
152 #}
153 #}
154
155 # Open the GTI log file for appending, or write to STDERR if that fails
156 if (!open(GTI_LOG, ">>$gti_log_file")) {
157 open(GTI_LOG, ">&STDERR");
158 }
159
160 # Log the command that launched this script
161 &log_message("Command: $0 @ARGV");
162
163 # Check that a command was supplied
164 if (!$gti_command) {
165 &throw_fatal_error("Missing command.");
166 }
167
168 # Process the command
169 if ($gti_command =~ /^get-all-chunks$/i) {
170 # Check that GS3 interface is the target
171 if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg, gs3deccolcfg
172 print &get_all_chunks_for_gs3(@gti_command_arguments);
173 } else {
174 print &get_all_chunks(@gti_command_arguments);
175 }
176 }
177 elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
178 if ($module =~ m/^gs3/) {
179 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
180 } else {
181 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
182 }
183 }
184 elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
185 if ($module =~ m/^gs3/) {
186 print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
187 } else {
188 print &get_uptodate_chunks(@gti_command_arguments);
189 }
190 }
191 elsif ($gti_command =~ /^get-language-status$/i) {
192 print &get_language_status(@gti_command_arguments);
193 }
194 elsif ($gti_command =~ /^search-chunks$/i) {
195 print &search_chunks(@gti_command_arguments);
196 }
197 elsif ($gti_command =~ /^submit-translations$/i) {
198 # This command cannot produce any output since it reads input
199 &submit_translations(@gti_command_arguments);
200 }
201 elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
202 # This command cannot produce any output since it reads input
203 &create_glihelp_zip_file(@gti_command_arguments);
204 }
205 else {
206 # The command was not recognized
207 &throw_fatal_error("Unknown command \"$gti_command\".");
208 }
209}
210
211
212sub throw_fatal_error
213{
214 my $error_message = shift(@_);
215
216 # Write an XML error response
217 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
218 print "<GTIResponse>\n";
219 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
220 print "</GTIResponse>\n";
221
222 # Log the error message, then die
223 &log_message("Error: $error_message");
224 die "\n";
225}
226
227
228sub log_message
229{
230 my $log_message = shift(@_);
231 print GTI_LOG time() . " -- " . $log_message . "\n";
232}
233
234
235sub get_all_chunks
236{
237 # The code of the target language (ensure it is lowercase)
238 my $target_language_code = lc(shift(@_));
239 # The key of the file to translate (ensure it is lowercase)
240 my $translation_file_key = lc(shift(@_));
241
242 # Check that the necessary arguments were supplied
243 if (!$target_language_code || !$translation_file_key) {
244 &throw_fatal_error("Missing command argument.");
245 }
246
247 # Get (and check) the translation configuration
248 my ($source_file, $target_file, $translation_file_type)
249 = &get_translation_configuration($target_language_code, $translation_file_key);
250
251 # Parse the source language and target language files
252 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
253 my @source_file_lines = &read_file_lines($source_file_path);
254 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
255
256 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
257 my @target_file_lines = &read_file_lines($target_file_path);
258 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
259
260 # Filter out any automatically translated chunks
261 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
262 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
263 delete $source_file_key_to_line_mapping{$chunk_key};
264 delete $target_file_key_to_line_mapping{$chunk_key};
265 }
266 }
267
268 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
269 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
270 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
271 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
272
273 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
274 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
275
276 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
277
278 return $xml_response;
279}
280
281
282sub get_uptodate_chunks
283{
284 # The code of the target language (ensure it is lowercase)
285 my $target_language_code = lc(shift(@_));
286 # The key of the file to translate (ensure it is lowercase)
287 my $translation_file_key = lc(shift(@_));
288
289 # Check that the necessary arguments were supplied
290 if (!$target_language_code || !$translation_file_key) {
291 &throw_fatal_error("Missing command argument.");
292 }
293
294 # Get (and check) the translation configuration
295 my ($source_file, $target_file, $translation_file_type)
296 = &get_translation_configuration($target_language_code, $translation_file_key);
297
298 # Parse the source language and target language files
299 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
300 my @source_file_lines = &read_file_lines($source_file_path);
301 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
302
303 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
304 my @target_file_lines = &read_file_lines($target_file_path);
305 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
306
307 # Filter out any automatically translated chunks
308 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
309 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
310 delete $source_file_key_to_line_mapping{$chunk_key};
311 delete $target_file_key_to_line_mapping{$chunk_key};
312 }
313 }
314
315 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
316 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
317 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
318 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
319
320 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
321 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
322
323
324 # Chunks needing updating are those in the target file that have been more recently edited in the source file
325 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
326 my @uptodate_target_file_keys = ();
327 foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
328 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
329 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
330
331 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
332
333 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
334 # &log_message("Chunk with key $chunk_key needs updating.");
335 push(@uptodate_target_file_keys, $chunk_key);
336 }
337 }
338
339 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
340
341 return $xml_response;
342}
343
344
345sub get_first_n_chunks_requiring_work
346{
347 # The code of the target language (ensure it is lowercase)
348 my $target_language_code = lc(shift(@_));
349 # The key of the file to translate (ensure it is lowercase)
350 my $translation_file_key = lc(shift(@_));
351 # The number of chunks to return (defaults to one if not specified)
352 my $num_chunks_to_return = shift(@_) || "1";
353
354 # Check that the necessary arguments were supplied
355 if (!$target_language_code || !$translation_file_key) {
356 &throw_fatal_error("Missing command argument.");
357 }
358
359 # Get (and check) the translation configuration
360 my ($source_file, $target_file, $translation_file_type)
361 = &get_translation_configuration($target_language_code, $translation_file_key);
362
363 # Parse the source language and target language files
364 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
365 my @source_file_lines = &read_file_lines($source_file_path);
366 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
367
368 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
369 my @target_file_lines = &read_file_lines($target_file_path);
370 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
371
372 # Filter out any automatically translated chunks
373 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
374 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
375 delete $source_file_key_to_line_mapping{$chunk_key};
376 delete $target_file_key_to_line_mapping{$chunk_key};
377 }
378 }
379
380 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
381 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
382 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
383 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
384
385 # Determine the target file chunks requiring translation
386 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
387 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
388
389 # Determine the target file chunks requiring updating
390 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
391 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
392 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
393 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
394
395 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
396
397 return $xml_response;
398}
399
400
401sub get_language_status
402{
403 # The code of the target language (ensure it is lowercase)
404 my $target_language_code = lc(shift(@_));
405
406 # Check that the necessary arguments were supplied
407 if (!$target_language_code) {
408 &throw_fatal_error("Missing command argument.");
409 }
410
411 # Form an XML response to the command
412 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
413 $xml_response .= "<GTIResponse>\n";
414 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
415
416 foreach my $translation_file (@$gti_translation_files) {
417 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
418 my $target_file_name = "";
419
420 if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg, gs3deccolcfg
421 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
422 &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
423
424 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
425 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
426
427 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
428 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
429 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
430 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
431 }
432 else {
433 # Get (and check) the translation configuration
434 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
435 $target_file_name = $target_file;
436
437 # Parse the source language and target language files
438 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
439 my @source_file_lines = &read_file_lines($source_file_path);
440 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
441
442 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
443 my @target_file_lines = &read_file_lines($target_file_path);
444 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
445
446 # Filter out any automatically translated chunks
447 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
448 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
449 delete $source_file_key_to_line_mapping{$chunk_key};
450 delete $target_file_key_to_line_mapping{$chunk_key};
451 }
452 }
453
454 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
455 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
456
457 # Determine the target file chunks requiring translation
458 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
459
460 # Determine the target file chunks requiring updating
461 my @target_file_keys_requiring_updating = ();
462 if (-e $target_file_path) {
463 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
464 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
465 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
466 }
467
468 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
469 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
470 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
471 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
472 }
473
474 &log_message("Status of " . $translation_file->{'key'});
475 &log_message("Number of source chunks: " . $num_source_chunks);
476 &log_message("Number of target chunks: " . $num_target_chunks);
477 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
478 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
479
480 $xml_response .= " <TranslationFile"
481 . " key=\"" . $translation_file->{'key'} . "\""
482 . " target_file_path=\"" . $target_file_name . "\""
483 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
484 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
485 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
486 }
487
488 $xml_response .= " </LanguageStatus>\n";
489
490 $xml_response .= "</GTIResponse>\n";
491 return $xml_response;
492}
493
494
495sub search_chunks
496{
497 # The code of the target language (ensure it is lowercase)
498 my $target_language_code = lc(shift(@_));
499 # The key of the file to translate (ensure it is lowercase)
500 my $translation_file_key = lc(shift(@_));
501 # The query string
502 my $query_string = join(' ', @_);
503
504 # Check that the necessary arguments were supplied
505 if (!$target_language_code || !$translation_file_key || !$query_string) {
506 &throw_fatal_error("Missing command argument.");
507 }
508
509 my ($source_file, $target_file, $translation_file_type) = ();
510 my %source_file_key_to_text_mapping = ();
511 my %target_file_key_to_text_mapping = ();
512
513
514 if ($translation_file_key !~ m/^gs3/) {
515 # Get (and check) the translation configuration
516 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
517
518 # Parse the source language and target language files
519 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
520 my @source_file_lines = &read_file_lines($source_file_path);
521 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
522
523 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
524 my @target_file_lines = &read_file_lines($target_file_path);
525 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
526
527 # Filter out any automatically translated chunks
528 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
529 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
530 delete $source_file_key_to_line_mapping{$chunk_key};
531 delete $target_file_key_to_line_mapping{$chunk_key};
532 }
533 }
534
535 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
536 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
537 }
538 else {
539 # Not needed in this case
540 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
541 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
542 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
543 }
544
545 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
546 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
547
548 # Determine the target file chunks matching the query
549 my @target_file_keys_matching_query = ();
550 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
551 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
552 #&log_message("**** Next chunk to compare search term $query_string to: $target_file_text.");
553 if ($target_file_text =~ /$query_string/i) {
554 # &log_message("Chunk with key $chunk_key matches query.");
555 push(@target_file_keys_matching_query, $chunk_key);
556 }
557 }
558
559 # Form an XML response to the command
560 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
561 $xml_response .= "<GTIResponse>\n";
562
563 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
564 foreach my $chunk_key (@target_file_keys_matching_query) {
565 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
566
567 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
568 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
569 $xml_response .= " </Chunk>\n";
570 }
571 $xml_response .= " </ChunksMatchingQuery>\n";
572
573 $xml_response .= "</GTIResponse>\n";
574 return $xml_response;
575}
576
577
578sub submit_translations
579{
580 # The code of the target language (ensure it is lowercase)
581 my $target_language_code = lc(shift(@_));
582 # The key of the file to translate (ensure it is lowercase)
583 my $translation_file_key = lc(shift(@_));
584 # The username of the translation submitter
585 my $submitter_username = shift(@_);
586 # Whether to submit a target chunk even if it hasn't changed
587 my $force_submission_flag = shift(@_);
588
589 # Check that the necessary arguments were supplied
590 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
591 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
592 die "\n";
593 }
594
595 my %source_file_key_to_text_mapping = ();
596 my %source_file_key_to_gti_comment_mapping = ();
597 my %target_file_key_to_text_mapping = ();
598 my %target_file_key_to_gti_comment_mapping = ();
599
600 my (@source_file_lines, @target_file_lines) = ();
601 my ($source_file, $target_file, $translation_file_type);
602
603
604 if ($translation_file_key !~ m/^gs3/) {
605 # Get (and check) the translation configuration
606 ($source_file, $target_file, $translation_file_type)
607 = &get_translation_configuration($target_language_code, $translation_file_key);
608
609 #&log_message("********************** START SOURCE ***********************************");
610
611 # Parse the source language and target language files
612 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
613 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
614 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
615 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
616
617 #&log_message("********************** START TARGET ***********************************");
618
619 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
620 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
621 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
622 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
623 }
624 else {
625 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
626 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
627 }
628 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
629 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
630
631 # Submission date
632 my $day = (localtime)[3];
633 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
634 my $year = (localtime)[5] + 1900;
635 my $submission_date = "$day-$month-$year";
636
637 open(SUBMISSION, "-");
638 my @submission_lines = <SUBMISSION>;
639 close(SUBMISSION);
640
641 # Remove any nasty carriage returns
642 # &log_message("Submission:");
643 foreach my $submission_line (@submission_lines) {
644 $submission_line =~ s/\r$//;
645 $submission_line =~ s/ +$//;
646 #&log_message(" $submission_line");
647 }
648
649 my %source_file_key_to_submission_mapping = ();
650 my %target_file_key_to_submission_mapping = ();
651 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
652 # Read source file part of submission
653 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
654 my $chunk_key = $1;
655
656 # Read the source file text
657 my $source_file_chunk_text = "";
658 $i++;
659 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
660 $source_file_chunk_text .= $submission_lines[$i];
661 $i++;
662 }
663 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
664 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
665
666 #&log_message("Source file key: |$chunk_key|");
667 #&log_message("Source file text: |$source_file_chunk_text|");
668 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
669 }
670
671 # Read target file part of submission
672 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
673 my $chunk_key = $1;
674
675 # Read the target file text
676 my $target_file_chunk_text = "";
677 $i++;
678 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
679 $target_file_chunk_text .= $submission_lines[$i];
680 $i++;
681 }
682 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
683 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
684
685 #&log_message("Target file key: |$chunk_key|");
686 #&log_message("Target file text: |$target_file_chunk_text|");
687 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
688 }
689 }
690
691 # -----------------------------------------
692 # Validate the translation submissions
693 # -----------------------------------------
694
695 # Check that the translations are valid
696 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
697
698 # Kathy introduced escaped colons ("\:") into chunk keys in properties files (greenstone3/metadata_names),
699 # but they're not escaped in the submitted XML versions, nor are they escaped in memory (in the $chunk_key)
700
701 # Make sure the submitted chunk still exists in the source file
702 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
703 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
704 delete $source_file_key_to_submission_mapping{$chunk_key};
705 delete $target_file_key_to_submission_mapping{$chunk_key};
706 next;
707 }
708 ## DEBUG
709 # if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
710 # &log_message("**** ORIG SRC: " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n");
711 # &log_message("**** SUBMIT SRC: " . &unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) . "\n");
712 # } else {
713 # &log_message("unmake-XML-safe versions of source chunk before and for submission match");
714 # }
715 ## END DEBUG
716
717 # Make sure the submitted source chunk matches the source file chunk
718 if ($source_file_key_to_submission_mapping{$chunk_key} ne $source_file_key_to_text_mapping{$chunk_key}
719 && $source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
720 #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
721 #&log_message("**** submission source:\n|$source_file_key_to_submission_mapping{$chunk_key}|\n");
722 #&log_message("**** unmake xml safe source:\n|" . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."|\n");
723
724 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
725 &log_message("Submission source: |$source_file_key_to_submission_mapping{$chunk_key}|");
726 &log_message(" Source text: |$source_file_key_to_text_mapping{$chunk_key}|");
727 delete $source_file_key_to_submission_mapping{$chunk_key};
728 delete $target_file_key_to_submission_mapping{$chunk_key};
729 next;
730 }
731 }
732
733 # Apply the submitted translations
734 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
735 # Only apply the submission if it is a change, unless -force_submission has been specified
736 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
737 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
738 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
739 }
740 }
741
742 if ($translation_file_key !~ m/^gs3/) {
743 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
744 } else {
745 eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
746 }
747}
748
749
750sub create_glihelp_zip_file
751{
752 my $target_language_code = shift(@_);
753 my $translation_file_key = "glihelp";
754
755 &log_message("Creating GLI Help zip file for $target_language_code");
756
757 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
758
759 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
760 my $oldclasspath = $classpath;
761 if ( ! -e $classpath) {
762 $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
763 }
764 if ( ! -e $classpath) {
765 &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
766 }
767
768
769 my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
770 my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
771 my $os = $^O;
772 my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
773 my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
774 $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
775
776 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
777 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
778
779 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
780 if ( ! -e $gen_many_html_xsl_filepath) {
781 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
782 }
783
784 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
785 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
786
787 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
788 $target_file_directory = $target_file_directory."/";
789
790 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
791
792 # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
793 my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
794 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
795
796 # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
797 my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
798 my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
799 &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
800
801 # The following file reading section is a candidate to use FileUtils::readUTF8File()
802 # in place of calling sysread() directly. But only if we can reason we'd be working with UTF8
803 # In gli/help/<newlang>/help.xml, replace all occurrences of
804 # <Text id="1">This text in en will be removed for new langcode</Text>
805 # with <!-- Missing translation: 1 -->
806 open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
807 my $help_xml_contents;
808 # Read in the entire contents of the file in one hit
809 sysread(FIN, $help_xml_contents, -s FIN);
810 close(FIN);
811
812 $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
813
814 open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
815 print FOUT $help_xml_contents;
816 close(FOUT);
817 }
818
819 my $perl_exec = &util::get_perl_exec();
820
821 my $java_exec = undef;
822 if (defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne "") {
823 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
824 }
825 elsif (defined($ENV{'JRE_HOME'}) && $ENV{'JRE_HOME'} ne "") {
826 $java_exec = &util::filename_cat($ENV{'JRE_HOME'}, "bin", "java");
827 }
828 else {
829 # Assume it is on the user's PATH
830 $java_exec = "java";
831 }
832
833 #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
834 my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
835 #&throw_fatal_error("RAN gti command: $cmd");
836 my $response = `$cmd`;
837
838 #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
839 $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
840 $response = `$cmd`;
841
842 # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
843 my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
844 if(!&FileUtils::directoryExists($tmpdir)) {
845 &FileUtils::makeDirectory($tmpdir);
846 }
847 #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
848 my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
849 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
850
851 $response = `$cmd`;
852}
853
854
855sub get_translation_configuration
856{
857 # Get the code of the target language
858 my $target_language_code = shift(@_);
859 # Get the key of the file to translate
860 my $translation_file_key = shift(@_);
861
862 # Read the translation data from the gti.cfg file
863 my ($source_file, $target_file, $translation_file_type) =
864 &get_translation_data_for($target_language_code, $translation_file_key);
865
866 # Check that the file to translate is defined in the gti.cfg file
867 if (!$source_file || !$target_file || !$translation_file_type) {
868 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
869 }
870
871 # Check that the source file exists
872 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
873 if (!-e $source_file_path) {
874 &throw_fatal_error("Source file $source_file_path does not exist.");
875 }
876
877 # Check that the source file is up to date
878 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
879 # unless ($translation_file_is_not_in_cvs) {
880 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
881 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
882 if ($source_file_cvs_status =~ /^C /) {
883 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
884 }
885 if ($source_file_cvs_status =~ /^M /) {
886 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
887 }
888 # }
889
890 return ($source_file, $target_file, $translation_file_type);
891}
892
893
894sub get_translation_data_for
895{
896 my ($target_language_code, $translation_file_key) = @_;
897
898 foreach my $translation_file (@$gti_translation_files) {
899 # If this isn't the correct translation file, move onto the next one
900 next if ($translation_file_key ne $translation_file->{'key'});
901
902 # Resolve the target language file
903 my $target_language_file = $translation_file->{'target_file'};
904 if ($target_language_file =~ /(\{.+\;.+\})/) {
905 my $unresolved_target_language_file_part = $1;
906
907 # Check for a special case for the target language code
908 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
909 my $resolved_target_language_file_part = $2;
910 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
911 }
912 # Otherwise use the last part as the default value
913 else {
914 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
915 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
916 }
917 }
918
919 # Resolve instances of {iso_639_1_target_language_name}
920 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
921 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
922 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
923
924 # Resolve instances of {target_language_code}
925 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
926
927 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
928}
929
930return ();
931}
932
933
934sub read_file_lines
935{
936 my ($file_path) = @_;
937
938 if (!open(FILE_IN, "<$file_path")) {
939 &log_message("Note: Could not open file $file_path.");
940 return ();
941 }
942 my @file_lines = <FILE_IN>;
943 close(FILE_IN);
944
945 return @file_lines;
946}
947
948
949sub build_key_to_line_mapping
950{
951 my ($file_lines, $translation_file_type) = @_;
952 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
953}
954
955
956sub build_key_to_text_mapping
957{
958 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
959
960 my %key_to_text_mapping = ();
961 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
962 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
963 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
964
965 my $chunk_text = @$file_lines[$chunk_starting_line];
966 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
967 $chunk_text .= @$file_lines[$l];
968 }
969
970 # Map from chunk key to text
971 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
972
973 #&log_message("@@@ chunk key: $chunk_key");
974 #&log_message("@@@ source string: |" . $key_to_text_mapping{$chunk_key} . "|");
975
976
977 #if($chunk_key =~ m/document\\/) {
978 #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
979 #&log_message("@@@ chunk key: $chunk_key");
980 #}
981
982 }
983
984 return %key_to_text_mapping;
985}
986
987
988sub build_key_to_last_update_date_mapping
989{
990 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
991
992 # If the files aren't in CVS then we can't tell anything about what needs updating
993 # return () if ($translation_file_is_not_in_cvs);
994
995 # Build a mapping from key to CVS date
996 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
997 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
998
999 # Build a mapping from key to comment date
1000 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
1001
1002 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
1003 my %key_to_last_update_date_mapping = ();
1004 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1005 # Use the CVS date as a starting point
1006 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
1007 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
1008
1009 # If a comment date exists and it is after the CVS date, use that instead
1010 # need to convert the comment date format to SVN format
1011 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
1012 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
1013 my $chunk_comment_date = $1;
1014 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
1015 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
1016 }
1017 }
1018 }
1019
1020 return %key_to_last_update_date_mapping;
1021}
1022
1023
1024sub build_key_to_cvs_date_mapping
1025{
1026 my ($filename, $translation_file_type) = @_;
1027
1028 # Use SVN to annotate each line of the file with the date it was last edited
1029 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
1030 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
1031
1032 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
1033
1034 my @cvs_annotated_file_lines_date = ();
1035 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
1036 # Extract the date from the SVN annotation at the front
1037 # svn format : 2007-07-16
1038 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
1039
1040 push(@cvs_annotated_file_lines_date, $1);
1041
1042 # trim extra date information in svn annotation format
1043 # 15:42:49 +1200 (Wed, 21 Jun 2006)
1044 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1045 }
1046
1047 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1048 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1049
1050 my %key_to_cvs_date_mapping = ();
1051 foreach my $chunk_key (keys(%key_to_line_mapping)) {
1052 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1053 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1054
1055 # Find the date this chunk was last edited, from the CVS annotation
1056 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
1057 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1058 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1059 # This part of the chunk has been updated more recently
1060 $chunk_date = $cvs_annotated_file_lines_date[$l];
1061
1062 }
1063 }
1064
1065 # Map from chunk key to CVS date
1066 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1067 }
1068
1069 return %key_to_cvs_date_mapping;
1070}
1071
1072
1073sub build_key_to_gti_comment_mapping
1074{
1075 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1076
1077 my %key_to_gti_comment_mapping = ();
1078 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1079 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1080 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1081
1082 my $chunk_text = @$file_lines[$chunk_starting_line];
1083 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1084 $chunk_text .= @$file_lines[$l];
1085 }
1086
1087 # Map from chunk key to GTI comment
1088 my $chunk_gti_comment;
1089 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1090 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1091 }
1092
1093 return %key_to_gti_comment_mapping;
1094}
1095
1096
1097sub determine_chunks_requiring_translation
1098{
1099 my $source_file_key_to_text_mapping = shift(@_);
1100 my $target_file_key_to_text_mapping = shift(@_);
1101
1102 # Chunks needing translation are those in the source file with no translation in the target file
1103 my @target_file_keys_requiring_translation = ();
1104 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1105 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1106 # &log_message("Chunk with key $chunk_key needs translating.");
1107 push(@target_file_keys_requiring_translation, $chunk_key);
1108 }
1109 }
1110
1111 return @target_file_keys_requiring_translation;
1112}
1113
1114
1115sub determine_chunks_requiring_updating
1116{
1117 my $source_file_key_to_last_update_date_mapping = shift(@_);
1118 my $target_file_key_to_last_update_date_mapping = shift(@_);
1119
1120 # Chunks needing updating are those in the target file that have been more recently edited in the source file
1121 my @target_file_keys_requiring_updating = ();
1122 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1123 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1124 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1125
1126 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1127
1128 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1129 # &log_message("Chunk with key $chunk_key needs updating.");
1130 # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1131 push(@target_file_keys_requiring_updating, $chunk_key);
1132 }
1133 }
1134
1135 return @target_file_keys_requiring_updating;
1136}
1137
1138
1139sub is_chunk_automatically_translated
1140{
1141 my ($chunk_key, $translation_file_type) = @_;
1142 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1143}
1144
1145
1146sub make_text_xml_safe
1147{
1148 my $text = shift(@_);
1149 $text =~ s/\&/\&amp\;/g;
1150 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1151 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1152 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1153 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1154 $text =~ s/</\&lt\;/g;
1155 $text =~ s/>/\&gt\;/g;
1156 return $text;
1157}
1158
1159
1160sub unmake_text_xml_safe
1161{
1162 my $text = shift(@_);
1163 $text =~ s/\&lt\;/</g;
1164 $text =~ s/\&gt\;/>/g;
1165 $text =~ s/\&amp\;/\&/g;
1166 return $text;
1167}
1168
1169
1170# Returns 1 if $date1 is after $date2, 0 otherwise
1171sub is_date_after_cvs
1172{
1173 my ($date1, $date2) = @_;
1174 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
1175 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1176
1177 if(!defined $date1) {
1178 return 1;
1179 }
1180
1181 my @date1parts = split(/-/, $date1);
1182 my @date2parts = split(/-/, $date2);
1183
1184 # Compare year - nasty because we have rolled over into a new century
1185 my $year1 = $date1parts[2];
1186 if ($year1 < 80) {
1187 $year1 += 2000;
1188 }
1189 my $year2 = $date2parts[2];
1190 if ($year2 < 80) {
1191 $year2 += 2000;
1192 }
1193
1194 # Compare year
1195 if ($year1 > $year2) {
1196 return 1;
1197 }
1198 elsif ($year1 == $year2) {
1199 # Year is the same, so compare month
1200 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1201 return 1;
1202 }
1203 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1204 # Month is the same, so compare day
1205 if ($date1parts[0] > $date2parts[0]) {
1206 return 1;
1207 }
1208 }
1209 }
1210
1211 return 0;
1212}
1213
1214sub is_date_after
1215{
1216 my ($date1, $date2) = @_;
1217
1218 if(!defined $date1) {
1219 return 1;
1220 }
1221 if(!defined $date2) {
1222 return 0;
1223 }
1224
1225 # 16-Aug-2006
1226 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1227 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1228 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1229 $date1=$3 . "-" . $months{$2} . "-" . $1;
1230 # print "** converted date1: $date1\n";
1231 }
1232 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1233 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1234 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1235 $date2=$3 . "-" . $months{$2} . "-" . $1;
1236 # print "** converted date2: $date2\n";
1237 }
1238
1239
1240 # 2006-08-16
1241 my @date1parts = split(/-/, $date1);
1242 my @date2parts = split(/-/, $date2);
1243
1244 # Compare year
1245 if ($date1parts[0] > $date2parts[0]) {
1246 return 1;
1247 }
1248 elsif ($date1parts[0] == $date2parts[0]) {
1249 # Year is the same, so compare month
1250 if ($date1parts[1] > $date2parts[1]) {
1251 return 1;
1252 }
1253 elsif ($date1parts[1] == $date2parts[1]) {
1254 # Month is the same, so compare day
1255 if ($date1parts[2] > $date2parts[2]) {
1256 return 1;
1257 }
1258 }
1259 }
1260
1261 return 0;
1262}
1263
1264
1265sub create_xml_response_for_chunks_requiring_work
1266{
1267 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1268
1269 # Form an XML response to the command
1270 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1271 $xml_response .= "<GTIResponse>\n";
1272 $xml_response .= " <TranslationFile"
1273 . " key=\"" . $translation_file_key . "\""
1274 . " target_file_path=\"" . $target_file . "\""
1275 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1276 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1277 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1278
1279 # Do chunks requiring translation first
1280 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1281 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1282 }
1283 else {
1284 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1285 }
1286
1287 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1288 foreach my $chunk_key (@sorted_chunk_keys) {
1289 last if ($num_chunks_to_return == 0);
1290
1291 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1292 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1293
1294 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1295 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1296 $xml_response .= " <TargetFileText></TargetFileText>\n";
1297 $xml_response .= " </Chunk>\n";
1298
1299 $num_chunks_to_return--;
1300 }
1301
1302 $xml_response .= " </ChunksRequiringTranslation>\n";
1303
1304 # Then do chunks requiring updating
1305 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1306 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1307 }
1308 else {
1309 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1310 }
1311
1312 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1313 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1314 foreach my $chunk_key (@sorted_chunk_keys) {
1315 last if ($num_chunks_to_return == 0);
1316
1317 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1318 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1319 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1320 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1321
1322 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1323 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1324 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1325 $xml_response .= " </Chunk>\n";
1326
1327 $num_chunks_to_return--;
1328 }
1329
1330 $xml_response .= " </ChunksRequiringUpdating>\n";
1331
1332 $xml_response .= "</GTIResponse>\n";
1333
1334 return $xml_response;
1335}
1336
1337sub create_xml_response_for_uptodate_chunks
1338{
1339 my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1340
1341 # Form an XML response to the command
1342 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1343 $xml_response .= "<GTIResponse>\n";
1344 $xml_response .= " <TranslationFile"
1345 . " key=\"" . $translation_file_key . "\""
1346 . " target_file_path=\"" . $target_file . "\""
1347 . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1348
1349
1350 # Then do chunks requiring updating
1351 $xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1352
1353
1354 # foreach my $chunk_key (@uptodate_target_file_keys) {
1355 my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1356 foreach my $chunk_key (@sorted_chunk_keys) {
1357
1358 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1359 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1360 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1361 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1362
1363 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1364 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1365 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1366 $xml_response .= " </Chunk>\n";
1367
1368 }
1369
1370 $xml_response .= " </UptodateChunks>\n";
1371
1372 $xml_response .= "</GTIResponse>\n";
1373
1374 return $xml_response;
1375}
1376
1377sub create_xml_response_for_all_chunks
1378{
1379 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1380
1381 # Form an XML response to the command
1382 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1383 $xml_response .= "<GTIResponse>\n";
1384 $xml_response .= " <TranslationFile"
1385 . " key=\"" . $translation_file_key . "\""
1386 . " target_file_path=\"" . $target_file . "\"\/>\n";
1387
1388 # Do all the chunks
1389 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1390
1391 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1392 foreach my $chunk_key (@sorted_chunk_keys) {
1393 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1394 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1395
1396 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1397 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1398 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1399 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1400 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1401 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1402 }
1403 else {
1404 $xml_response .= " <TargetFileText></TargetFileText>\n";
1405 }
1406
1407 $xml_response .= " </Chunk>\n";
1408 }
1409 $xml_response .= " </Chunks>\n";
1410
1411 $xml_response .= "</GTIResponse>\n";
1412 return $xml_response;
1413}
1414
1415
1416
1417# ==========================================================================================
1418# MACROFILE FUNCTIONS
1419
1420sub build_key_to_line_mapping_for_macrofile
1421{
1422 my (@file_lines) = @_;
1423
1424 my $macro_package;
1425 my %chunk_key_to_line_mapping = ();
1426 # Process the contents of the file, line by line
1427 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1428 my $line = $file_lines[$i];
1429 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1430
1431 # Check if a new package is being defined
1432 if ($line =~ m/^package\s+(.+)/) {
1433 $macro_package = $1;
1434 }
1435
1436 # Line contains a macro name
1437 elsif ($line =~ m/^(_\w+_)/) {
1438 my $macro_key = $1;
1439 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1440
1441 # While there is still text of the macro to go...
1442 my $startline = $i;
1443 while ($line !~ /\}$/) {
1444 $i++;
1445 if ($i == scalar(@file_lines)) {
1446 &throw_fatal_error("Could not find end of macro $macro_key.");
1447 }
1448 $line = $file_lines[$i];
1449 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1450 }
1451
1452 # The chunk key consists of the package name and the macro key
1453 my $chunk_key = $macro_package . "." . $macro_key;
1454 # Map from chunk key to line
1455 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1456 }
1457
1458 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1459 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1460 # The chunk key consists of package name and macro key
1461 my $chunk_key = $macro_package . "." . $1;
1462 # Map from chunk key to line
1463 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1464}
1465}
1466
1467return %chunk_key_to_line_mapping;
1468}
1469
1470
1471sub import_chunk_from_macrofile
1472{
1473 my ($chunk_text) = @_;
1474
1475 # Is this an icon macro??
1476 if ($chunk_text =~ /^\#\# (.*)/) {
1477 # Extract image macro text
1478 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1479 $chunk_text = $1;
1480
1481 # Remove enclosing quotes
1482 $chunk_text =~ s/^\"//;
1483 $chunk_text =~ s/\"$//;
1484 }
1485
1486 # No, so it must be a text macro
1487 else {
1488 # Remove macro key
1489 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1490
1491 # Remove language specifier
1492 $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1493
1494 # Remove braces enclosing text
1495 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1496 }
1497
1498 return $chunk_text;
1499}
1500
1501
1502sub get_macrofile_chunk_gti_comment
1503{
1504 my ($chunk_text) = @_;
1505
1506 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1507 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1508 return $1;
1509}
1510
1511return undef;
1512}
1513
1514
1515sub is_macrofile_chunk_automatically_translated
1516{
1517 my ($chunk_key) = @_;
1518
1519 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1520 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1521 return 1;
1522 }
1523
1524 return 0;
1525}
1526
1527
1528# Use the source file to generate a target file that is formatted the same
1529sub write_translated_macrofile
1530{
1531 my $source_file = shift(@_); # Not used
1532 my @source_file_lines = @{shift(@_)};
1533 my $source_file_key_to_text_mapping = shift(@_);
1534 my $target_file = shift(@_);
1535 my @target_file_lines = @{shift(@_)};
1536 my $target_file_key_to_text_mapping = shift(@_);
1537 my $target_file_key_to_gti_comment_mapping = shift(@_);
1538 my $target_language_code = shift(@_);
1539
1540 # Build a mapping from source file line to chunk key
1541 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1542 my %source_file_line_to_key_mapping = ();
1543 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1544 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1545 }
1546 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1547 my $source_file_line_number = 0;
1548
1549 # Build a mapping from target file line to chunk key
1550 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1551 my %target_file_line_to_key_mapping = ();
1552 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1553 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1554 }
1555 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1556
1557 # Write the new target file
1558 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1559 if (!open(TARGET_FILE, ">$target_file_path")) {
1560 &throw_fatal_error("Could not write target file $target_file_path.");
1561 }
1562
1563 # Use the header from the target file, to keep language and author information
1564 if (scalar(@target_file_line_keys) > 0) {
1565 my $target_file_line_number = 0;
1566 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1567 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1568 my $target_file_line = $target_file_lines[$target_file_line_number];
1569 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1570 print TARGET_FILE $target_file_line;
1571 $target_file_line_number++;
1572 }
1573
1574 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1575 }
1576
1577 # Model the new target file on the source file, with the target file translations
1578 foreach my $line_key (@source_file_line_keys) {
1579 # Fill in the gaps before this chunk starts
1580 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1581 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1582 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1583 print TARGET_FILE $source_file_lines[$source_file_line_number];
1584 $source_file_line_number++;
1585 }
1586 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1587
1588 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1589 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1590 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1591
1592 my $macrofile_key = $chunk_key;
1593 $macrofile_key =~ s/^(.+?)\.//;
1594
1595 # If no translation exists for this chunk, show this, and move on
1596 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1597 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1598 next;
1599 }
1600
1601 # Grab the source chunk text
1602 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1603 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1604 $source_file_chunk .= $source_file_lines[$l];
1605 }
1606
1607 # Is this an icon macro??
1608 if ($source_file_chunk =~ /^\#\# (.*)/) {
1609 # Escape any newline and question mark characters so the source text is replaced correctly
1610 $source_file_chunk_text =~ s/\\/\\\\/g;
1611 $source_file_chunk_text =~ s/\?/\\\?/g;
1612
1613 # Build the new target chunk from the source chunk
1614 my $target_file_chunk = $source_file_chunk;
1615 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1616 $target_file_chunk =~ s/(\s)*$//;
1617 print TARGET_FILE "$target_file_chunk";
1618 }
1619
1620 # No, it is just a normal text macro
1621 else {
1622 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1623 }
1624
1625 # Add the "updated" comment, if one exists
1626 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1627 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1628 }
1629 print TARGET_FILE "\n";
1630}
1631
1632close(TARGET_FILE);
1633}
1634
1635
1636sub sort_by_line
1637{
1638 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1639}
1640
1641
1642# ==========================================================================================
1643# RESOURCE BUNDLE FUNCTIONS
1644
1645# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1646sub build_key_to_line_mapping_for_resource_bundle
1647{
1648 my (@file_lines) = @_;
1649
1650 my %chunk_key_to_line_mapping = ();
1651
1652 my $chunk_key;
1653 my $startindex = -1;
1654
1655 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1656 my $line = $file_lines[$i];
1657 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1658 #&log_message("Got line: ".$line);
1659
1660 # a property line has a colon/equals sign as separator that is NOT escaped with a backslash (both keys and values
1661 # can use the colon or = sign. But in the key, such a char is always escaped. Unfortunately, they've not always been
1662 # escaped in the values. So we get the left most occurrence by not doing a greedy match (use ? to not be greedy).
1663 # So find the first :/= char not preceded by \. That will be the true separator of a chunk_key and its value chunk_text
1664
1665 if ($line =~ m/^(\S*?[^\\])[:|=](.*)$/) {
1666 # Line contains a dictionary string
1667
1668 # Unused but useful: http://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
1669 # http://perldoc.perl.org/perlvar.html
1670
1671 $chunk_key = $1;
1672 # remove the escaping of any :/= property separator from the chunk_key in memory,
1673 # to make comparison with its unescaped version during submissions easier. Will write out with escaping.
1674 $chunk_key =~ s/\\([:=])/$1/g;
1675
1676 $startindex = $i;
1677 }
1678 if ($startindex != -1) {
1679 if($line !~ m/\\$/) { # line finished
1680 # $i keeps track of the line at which this property (chunk_key) finishes
1681
1682 # Map from chunk key to line
1683 $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1684 $startindex = -1;
1685 $chunk_key = "";
1686 }
1687 }
1688 }
1689
1690 return %chunk_key_to_line_mapping;
1691}
1692
1693
1694sub import_chunk_from_resource_bundle
1695{
1696 my ($chunk_text) = @_;
1697
1698 # Simple: just remove string key.
1699 # But key can contain an escaped separator (\: or \=).
1700 # So just as in the previous subroutine, find the first (leftmost) : or = char not preceded by \.
1701 # That will be the true separator of a chunk_key and its value chunk_text
1702 $chunk_text =~ s/^(\S*?[^\\])[:|=](\s*)//s;
1703
1704 $chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
1705 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1706
1707 return $chunk_text;
1708}
1709
1710
1711sub get_resource_bundle_chunk_gti_comment
1712{
1713 my ($chunk_text) = @_;
1714
1715 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1716 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1717 return $1;
1718 }
1719
1720 return undef;
1721}
1722
1723
1724sub is_resource_bundle_chunk_automatically_translated
1725{
1726 # No resource bundle chunks are automatically translated
1727 return 0;
1728}
1729
1730
1731sub write_translated_resource_bundle
1732{
1733 my $source_file = shift(@_); # Not used
1734 my @source_file_lines = @{shift(@_)};
1735 my $source_file_key_to_text_mapping = shift(@_);
1736 my $target_file = shift(@_);
1737 my @target_file_lines = @{shift(@_)}; # Not used
1738 my $target_file_key_to_text_mapping = shift(@_);
1739 my $target_file_key_to_gti_comment_mapping = shift(@_);
1740 my $target_language_code = shift(@_); # Not used
1741
1742 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1743 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1744 my %source_file_line_to_key_mapping = ();
1745 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1746 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1747 }
1748
1749 # Write the new target file
1750 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1751 if (!open(TARGET_FILE, ">$target_file_path")) {
1752 &throw_fatal_error("Could not write target file $target_file_path.");
1753 }
1754
1755 # Model the new target file on the source file, with the target file translations
1756 my $source_file_line_number = 0;
1757 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1758 # Fill in the gaps before this chunk starts
1759 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1760 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1761 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1762 print TARGET_FILE $source_file_lines[$source_file_line_number];
1763 $source_file_line_number++;
1764 }
1765 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1766
1767 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1768 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1769 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1770
1771 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
1772 # since the key-value separator in a property resource bundle file is : or =
1773 my $escaped_chunk_key = $chunk_key;
1774 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
1775
1776 # If no translation exists for this chunk, show this, and move on
1777 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1778 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
1779 next;
1780 }
1781
1782 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
1783 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1784 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1785 }
1786 print TARGET_FILE "\n";
1787 }
1788
1789 close(TARGET_FILE);
1790}
1791
1792
1793# ==========================================================================================
1794# GREENSTONE XML FUNCTIONS
1795
1796sub build_key_to_line_mapping_for_greenstone_xml
1797{
1798 my (@file_lines) = @_;
1799
1800 my %chunk_key_to_line_mapping = ();
1801 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1802 my $line = $file_lines[$i];
1803 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1804 #&log_message("Got line: ".$line);
1805 # Line contains a string to translate
1806 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1807 my $chunk_key = $1;
1808 $line =~ s/\s*$//; # Remove any nasty whitespace
1809 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1810
1811 # While there is still text of the string to go...
1812 my $startline = $i;
1813 while ($line !~ /<\/Text>\s*$/) {
1814 $i++;
1815 if ($i == scalar(@file_lines)) {
1816 &throw_fatal_error("Could not find end of string $chunk_key.");
1817 }
1818 $line = $file_lines[$i];
1819 $line =~ s/\s*$//; # Remove any nasty whitespace
1820 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1821 }
1822 #&log_message("@@@ Line is now: ".$line);
1823 # Map from chunk key to line
1824 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1825 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1826 }
1827 else {
1828 &throw_fatal_error("Duplicate key $chunk_key.");
1829 }
1830 }
1831 }
1832
1833 return %chunk_key_to_line_mapping;
1834}
1835
1836
1837sub import_chunk_from_greenstone_xml
1838{
1839 my ($chunk_text) = @_;
1840
1841 # Simple: just remove the Text tags
1842 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1843 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1844 $chunk_text =~ s/\s*<\/Text>\s*$//;
1845
1846 return $chunk_text;
1847}
1848
1849
1850sub get_greenstone_xml_chunk_gti_comment
1851{
1852 my ($chunk_text) = @_;
1853
1854 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1855 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1856 return $1;
1857 }
1858
1859 return undef;
1860}
1861
1862
1863sub is_greenstone_xml_chunk_automatically_translated
1864{
1865 # No greenstone XML chunks are automatically translated
1866 return 0;
1867}
1868
1869
1870sub write_translated_greenstone_xml
1871{
1872 my $source_file = shift(@_); # Not used
1873 my @source_file_lines = @{shift(@_)};
1874 my $source_file_key_to_text_mapping = shift(@_);
1875 my $target_file = shift(@_);
1876 my @target_file_lines = @{shift(@_)}; # Not used
1877 my $target_file_key_to_text_mapping = shift(@_);
1878 my $target_file_key_to_gti_comment_mapping = shift(@_);
1879 my $target_language_code = shift(@_); # Not used
1880
1881 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1882 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1883 my %source_file_line_to_key_mapping = ();
1884 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1885 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1886 }
1887
1888 # Write the new target file
1889 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1890 if (!open(TARGET_FILE, ">$target_file_path")) {
1891 &throw_fatal_error("Could not write target file $target_file_path.");
1892 }
1893
1894 # Model the new target file on the source file, with the target file translations
1895 my $source_file_line_number = 0;
1896 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1897 # Fill in the gaps before this chunk starts
1898 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1899 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1900 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1901 print TARGET_FILE $source_file_lines[$source_file_line_number];
1902 $source_file_line_number++;
1903 }
1904 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1905
1906 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1907 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1908 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1909 $target_file_chunk_text =~ s/(\n)*$//g;
1910
1911 # If no translation exists for this chunk, show this, and move on
1912 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1913 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1914 next;
1915 }
1916
1917 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1918 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1919 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1920 $chunk_gti_comment =~ s/^Updated //;
1921 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1922 }
1923 print TARGET_FILE "\n";
1924 }
1925
1926 # Fill in the end of the file
1927 while ($source_file_line_number < scalar(@source_file_lines)) {
1928 print TARGET_FILE $source_file_lines[$source_file_line_number];
1929 $source_file_line_number++;
1930 }
1931
1932 close(TARGET_FILE);
1933}
1934
1935
1936# ==========================================================================================
1937# GREENSTONE3 FUNCTIONS
1938
1939sub get_all_chunks_for_gs3
1940{
1941 # The code of the target language (ensure it is lowercase)
1942 my $target_language_code = lc(shift(@_));
1943 my $translation_file_key = lc(shift(@_));
1944
1945 # Check that the necessary arguments were supplied
1946 if (!$target_language_code) {
1947 &throw_fatal_error("Missing command argument.");
1948 }
1949
1950 # Get (and check) the translation configuration
1951 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1952
1953 my %source_files_key_to_text_mapping = ();
1954 my %target_files_key_to_text_mapping = ();
1955 my %source_files_key_to_last_update_date_mapping = ();
1956 my %target_files_key_to_last_update_date_mapping = ();
1957
1958 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1959
1960 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1961 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1962
1963 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1964 return $xml_response;
1965}
1966
1967
1968sub get_first_n_chunks_requiring_work_for_gs3
1969{
1970 # The code of the target language (ensure it is lowercase)
1971 my $target_language_code = lc(shift(@_));
1972 # The key of the file to translate (ensure it is lowercase)
1973 my $translation_file_key = lc(shift(@_));
1974 # The number of chunks to return (defaults to one if not specified)
1975 my $num_chunks_to_return = shift(@_) || "1";
1976
1977 # Check that the necessary arguments were supplied
1978 if (!$target_language_code || !$translation_file_key) {
1979 &throw_fatal_error("Missing command argument.");
1980 }
1981
1982 my %source_files_key_to_text_mapping = ();
1983 my %target_files_key_to_text_mapping = ();
1984 my %source_files_key_to_last_update_date_mapping = ();
1985 my %target_files_key_to_last_update_date_mapping = ();
1986
1987 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1988 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1989
1990 # Determine the target file chunks requiring translation
1991 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1992 # Determine the target file chunks requiring updating
1993 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1994 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1995 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1996
1997 my $download_target_filepath = "";
1998
1999
2000 # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
2001
2002 # if there is no copy of the language files for download, there's also no link to the spreadsheet
2003 # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
2004 # interface *.properties files,and link to that zip as the file for offline translation.
2005 # Selecting only properties files for English and the language they're working on (if the last exists)
2006
2007 # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
2008 # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
2009
2010 my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
2011 my $tar_cmd = "tar -cvzf $zip";
2012
2013
2014 # store cur dir and cd to gsdlhome to generate the correct path in the zip file
2015 my $curdir = `pwd`;
2016 chdir $gsdl_root_directory;
2017
2018 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
2019 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
2020 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-dec-col-cfgs", \@gs3_dec_col_cfg_files);
2021
2022 # tar command will overwrite the previous version, but want to check we've created it
2023 if(&FileUtils::fileExists($zip)) {
2024 &FileUtils::removeFiles($zip);
2025 }
2026
2027 #my $tar_result = system($tar_cmd); # works but then interface breaks
2028 `$tar_cmd`;
2029 my $tar_result = $?;
2030
2031 if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
2032 $download_target_filepath = $zip;
2033 } else {
2034 &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
2035 }
2036
2037 # change back to original working directory (cgi-bin/linux probably)
2038 chdir $curdir;
2039
2040 # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
2041
2042
2043 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
2044 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
2045 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2046 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2047
2048 return $xml_response;
2049}
2050
2051# helper function
2052# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg, gs3deccolccfg)
2053# formatted correctly to go into a zip file
2054sub get_gs3_zip_file_listing
2055{
2056 my $target_language_code = shift(@_);
2057 my $sourcedir = shift(@_);
2058 my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
2059
2060 my $filelisting = "";
2061 foreach my $interface_file (@$files_array) {
2062
2063 my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
2064 my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2065
2066 $filelisting = "$filelisting $source_filepath";
2067 if(&FileUtils::fileExists($target_filepath)) {
2068 $filelisting = "$filelisting $target_filepath";
2069 }
2070 }
2071
2072 return $filelisting;
2073}
2074
2075sub get_uptodate_chunks_for_gs3
2076{
2077 # The code of the target language (ensure it is lowercase)
2078 my $target_language_code = lc(shift(@_));
2079 # The key of the file to translate (ensure it is lowercase)
2080 my $translation_file_key = lc(shift(@_));
2081 # The number of chunks to return (defaults to one if not specified)
2082 my $num_chunks_to_return = shift(@_) || "1";
2083
2084 # Check that the necessary arguments were supplied
2085 if (!$target_language_code || !$translation_file_key) {
2086 &throw_fatal_error("Missing command argument.");
2087 }
2088
2089 my %source_files_key_to_text_mapping = ();
2090 my %target_files_key_to_text_mapping = ();
2091 my %source_files_key_to_last_update_date_mapping = ();
2092 my %target_files_key_to_last_update_date_mapping = ();
2093
2094 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2095 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2096
2097
2098 # Chunks needing updating are those in the target file that have been more recently edited in the source file
2099 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2100 my @uptodate_target_file_keys = ();
2101 foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2102 my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2103 my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2104
2105 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
2106
2107 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2108 # &log_message("Chunk with key $chunk_key needs updating.");
2109 push(@uptodate_target_file_keys, $chunk_key);
2110 }
2111 }
2112
2113 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2114
2115 return $xml_response;
2116}
2117
2118
2119sub build_gs3_configuration
2120{
2121 my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2122 $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2123
2124 my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2125 my $files_array = \@gs3_interface_files;
2126
2127 if($translation_file_key eq "gs3colcfg") {
2128 $source_file_directory = "gs3-collection-configs";
2129 $files_array = \@gs3_col_cfg_files;
2130 } elsif($translation_file_key eq "gs3deccolcfg") {
2131 $source_file_directory = "gs3-dec-col-cfgs";
2132 $files_array = \@gs3_dec_col_cfg_files;
2133 }
2134 my $translation_file_type = "resource_bundle";
2135
2136 foreach my $interface_file_key (@$files_array) {
2137
2138 &log_message("Greenstone 3 interface file: " . $interface_file_key);
2139
2140 # Parse the source language and target language files
2141 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2142 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2143 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2144 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2145 #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2146
2147 my %source_file_key_to_gti_comment_or_last_updated_mapping;
2148 if($get_gti_comments_not_last_updated) {
2149 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2150 } else {
2151 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2152 }
2153
2154 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2155 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2156 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2157 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2158 #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2159
2160 my %target_file_key_to_gti_comment_or_last_updated_mapping;
2161 if($get_gti_comments_not_last_updated) {
2162 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2163 } else {
2164 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2165 }
2166
2167
2168 # Filter out any automatically translated chunks
2169 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2170 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2171 delete $source_file_key_to_line_mapping{$chunk_key};
2172 delete $target_file_key_to_line_mapping{$chunk_key};
2173 }
2174 }
2175
2176 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2177 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2178
2179 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2180 my $global_chunk_key = "$interface_file_key.$chunk_key";
2181 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2182 $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2183
2184 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2185 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2186 $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2187 }
2188 }
2189 }
2190}
2191
2192
2193sub write_translated_gs3interface
2194{
2195 my $translation_file_key = shift(@_);
2196 my $source_file_key_to_text_mapping = shift(@_);
2197 my $target_file_key_to_text_mapping = shift(@_);
2198 my $target_file_key_to_gti_comment_mapping = shift(@_);
2199 my $target_language_code = shift(@_);
2200
2201 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2202
2203 my %translated_interface_file_keys = ();
2204 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2205 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2206 if (!defined $translated_interface_file_keys{$1}) {
2207 &log_message("Updated interface file: " . $1);
2208 $translated_interface_file_keys{$1}="";
2209 }
2210 }
2211 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2212
2213 my $source_file_directory = "greenstone3";
2214 $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2215 $source_file_directory = "gs3-dec-col-cfgs" if $translation_file_key eq "gs3deccolcfg";
2216
2217 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2218
2219 # Build a mapping from chunk key to source file line, and from source file line to chunk key
2220 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2221 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2222 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2223 my %source_file_line_to_key_mapping = ();
2224 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2225 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2226 }
2227
2228 # Write the new target file
2229 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2230 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2231 if (!open(TARGET_FILE, ">$target_file_path")) {
2232 &throw_fatal_error("Could not write target file $target_file_path.");
2233 }
2234
2235 # Model the new target file on the source file, with the target file translations
2236 my $source_file_line_number = 0;
2237 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2238 # Fill in the gaps before this chunk starts
2239 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2240 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2241 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2242 print TARGET_FILE $source_file_lines[$source_file_line_number];
2243 $source_file_line_number++;
2244 }
2245 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2246
2247 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2248 my $global_chunk_key = "$interface_file_key.$chunk_key";
2249 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2250 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2251
2252 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
2253 # since the key-value separator in a property resource bundle file is : or =
2254 my $escaped_chunk_key = $chunk_key;
2255 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
2256
2257 # If no translation exists for this chunk, show this, and move on
2258 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2259 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
2260 next;
2261 }
2262
2263 if ($translation_file_key eq "gs3deccolcfg") { # leave in the equals sign as key-value separator in gs3deccolcfg module
2264 print TARGET_FILE "$escaped_chunk_key=$target_file_chunk_text";
2265 } else { # for other modules, do as before: use a colon as key-value separator
2266 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
2267 }
2268 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2269 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2270 }
2271 print TARGET_FILE "\n";
2272 }
2273
2274 close(TARGET_FILE);
2275 }
2276}
2277
2278&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.