source: main/trunk/greenstone2/bin/script/gti.pl

Last change on this file was 38761, checked in by anupama, 2 months ago

Changes to get the Italian translation spreadsheet files manually submitted without them throwing up so many warnings about submission source text being different from source text. Issues were 1. entity ampersand#10; (line feed character) was now followed by newline rather than followed by space when the unicode file came out of Excel, and it needed to be removed now. Excel also doesn't seem to add an extra space at the start. So we don't want to remove spaces at start where this exist in the original source string. Both were changes to gti-process-spreadsheet.pl. 2. Next, in gti.pl, a fix I had made in revision 33436 to remove extra spaces at line end that got added when files were processed for the glihelp module, would for regular properties files remove actual spaces at line end that marked submission source different from source. The earlier change may still be necessary for glihelp, so I now only remove that extra space if the module is glihelp.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 99.8 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en"; # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47
48# greenstone 3 interface files, from https://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
49# check it out as greenstone3
50{ 'key' => "gs3interface",
51 'file_type' => "resource_bundle",
52 'source_file' => "greenstone3",
53 'target_file' => "greenstone3"
54},
55
56# GLI dictionary
57{ 'key' => "glidict",
58 'file_type' => "resource_bundle",
59 'source_file' => "gli/classes/dictionary.properties",
60 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
61
62# Greenstone Installer interface
63{ 'key' => "gsinstaller",
64 'file_type' => "resource_bundle",
65 'source_file' => "gsinstaller/LanguagePack.properties",
66 'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
67
68# Greenstone Perl modules
69{ 'key' => "perlmodules",
70 'file_type' => "resource_bundle",
71 'source_file' => "perllib/strings.properties",
72 'target_file' => "perllib/strings_{target_language_code}.properties" },
73
74# GLI help
75{ 'key' => "glihelp",
76 'file_type' => "greenstone_xml",
77 'source_file' => "gli/help/en/help.xml",
78 'target_file' => "gli/help/{target_language_code}/help.xml" },
79
80# Greenstone 2 core
81{ 'key' => "coredm",
82 'file_type' => "macrofile",
83 'source_file' => "macros/english.dm",
84 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;sr-bh-cyr:serbian-bh-cyr;sr-bh-lat:serbian-bh-lat;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
85
86#Greenstone 2 aux
87{ 'key' => "auxdm",
88 'file_type' => "macrofile",
89 'source_file' => "macros/english2.dm",
90 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
91
92#{ 'key' => "paperspastdm",
93# 'file_type' => "macrofile",
94# 'source_file' => "macros/paperspast-english.dm",
95# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
96
97
98
99
100# Greenstone tutorial exercises
101# { 'key' => "tutorials",
102# 'file_type' => "greenstone_xml",
103# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
104# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
105
106# new Greenstone.org
107{ 'key' => "greenorg",
108 'file_type' => "resource_bundle",
109 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
110 'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
111},
112
113# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
114# from https://svn.greenstone.org/main/trunk/gs3-collection-configs
115{ 'key' => "gs3colcfg",
116 'file_type' => "resource_bundle",
117 'source_file' => "gs3-collection-configs",
118 'target_file' => "gs3-collection-configs"
119},
120
121
122# collection config display items of GS3 port of Documented Examples Collections (dec). Check out as gs3-dec-col-cfgs
123# from https://svn.greenstone.org/main/trunk/gs3-dec-col-cfgs
124{ 'key' => "gs3deccolcfg",
125 'file_type' => "resource_bundle",
126 'source_file' => "gs3-dec-col-cfgs",
127 'target_file' => "gs3-dec-col-cfgs"
128}
129];
130
131my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo", "localsite");
132
133my @gs3_dec_col_cfg_files = ("authen-e", "bibtex-e", "dls-e", "garish-e", "gsarch-e", "image-e", "isis-e", "lomdemo-e", "manifest-demo-e", "marc-e", "oai-e", "pagedimg-e", "style-e", "wiki-e", "wrdpdf-e"); # "bibtex-supp-e"
134
135my @gs3_interface_files = ("interface_default", "core_servlet_dictionary", "metadata_names");
136#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
137
138# Auxilliary GS3 interface files. This list is not used at present
139# Combine with above list if generating translation spreadsheet for all interface files
140my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
141
142my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
143
144# Not: i18n, log4j
145
146sub main
147{
148 # Get the command to process, and any arguments
149 my $gti_command = shift(@_);
150 my @gti_command_arguments = @_;
151 my $module = $_[1];
152
153 # for GS3, set gsdl_root_dir to GSDL3HOME
154 #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
155 #if($ENV{'GSDL3SRCHOME'}) {
156 # $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
157 # $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
158 #}
159 #}
160
161 # Open the GTI log file for appending, or write to STDERR if that fails
162 if (!open(GTI_LOG, ">>$gti_log_file")) {
163 open(GTI_LOG, ">&STDERR");
164 }
165
166 # Log the command that launched this script
167 &log_message("Command: $0 @ARGV");
168
169 # Check that a command was supplied
170 if (!$gti_command) {
171 &throw_fatal_error("Missing command.");
172 }
173
174 # For the first translation of glihelp into a new language, glihelp folders need to be
175 # initialised. The folders & files are not yet present on submit press, so do this then:
176 if($gti_command =~ /^submit-translations$/i && $module && $module eq "glihelp") {
177 my $langcode = shift(@_);
178 my $target_language_code = lc($langcode);
179 unshift(@_, $langcode); # put it back into args list
180
181 my $glihelp_lang_dir = &util::filename_cat($gsdl_root_directory, "gli", "help",$target_language_code);
182
183 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
184 # This command cannot produce any output since it reads input
185 &create_glihelp_zip_file(@gti_command_arguments);
186 }
187 #&log_message("@@@ Got cmd: $gti_command");
188 }
189
190 # Process the command
191 if ($gti_command =~ /^get-all-chunks$/i) {
192 # Check that GS3 interface is the target
193 if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg, gs3deccolcfg
194 print &get_all_chunks_for_gs3(@gti_command_arguments);
195 } else {
196 print &get_all_chunks(@gti_command_arguments);
197 }
198 }
199 elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
200 if ($module =~ m/^gs3/) {
201 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
202 } else {
203 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
204 }
205 }
206 elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
207 if ($module =~ m/^gs3/) {
208 print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
209 } else {
210 print &get_uptodate_chunks(@gti_command_arguments);
211 }
212 }
213 elsif ($gti_command =~ /^get-language-status$/i) {
214 print &get_language_status(@gti_command_arguments);
215 }
216 elsif ($gti_command =~ /^search-chunks$/i) {
217 print &search_chunks(@gti_command_arguments);
218 }
219 elsif ($gti_command =~ /^submit-translations$/i) {
220 # This command cannot produce any output since it reads input
221 &submit_translations(@gti_command_arguments);
222 }
223 elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
224 # This command cannot produce any output since it reads input
225 &create_glihelp_zip_file(@gti_command_arguments);
226 }
227 else {
228 # The command was not recognized
229 &throw_fatal_error("Unknown command \"$gti_command\".");
230 }
231}
232
233
234sub throw_fatal_error
235{
236 my $error_message = shift(@_);
237
238 # Write an XML error response
239 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
240 print "<GTIResponse>\n";
241 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
242 print "</GTIResponse>\n";
243
244 # Log the error message, then die
245 &log_message("Error: $error_message");
246 die "\n";
247}
248
249
250sub log_message
251{
252 my $log_message = shift(@_);
253 print GTI_LOG time() . " -- " . $log_message . "\n";
254}
255
256
257sub get_all_chunks
258{
259 # The code of the target language (ensure it is lowercase)
260 my $target_language_code = lc(shift(@_));
261 # The key of the file to translate (ensure it is lowercase)
262 my $translation_file_key = lc(shift(@_));
263
264 # Check that the necessary arguments were supplied
265 if (!$target_language_code || !$translation_file_key) {
266 &throw_fatal_error("Missing command argument.");
267 }
268
269 # Get (and check) the translation configuration
270 my ($source_file, $target_file, $translation_file_type)
271 = &get_translation_configuration($target_language_code, $translation_file_key);
272
273 # Parse the source language and target language files
274 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
275 my @source_file_lines = &read_file_lines($source_file_path);
276 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
277
278 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
279 my @target_file_lines = &read_file_lines($target_file_path);
280 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
281
282 # Filter out any automatically translated chunks
283 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
284 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
285 delete $source_file_key_to_line_mapping{$chunk_key};
286 delete $target_file_key_to_line_mapping{$chunk_key};
287 }
288 }
289
290 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
291 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
292 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
293 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
294
295 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
296 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
297
298 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
299
300 return $xml_response;
301}
302
303
304sub get_uptodate_chunks
305{
306 # The code of the target language (ensure it is lowercase)
307 my $target_language_code = lc(shift(@_));
308 # The key of the file to translate (ensure it is lowercase)
309 my $translation_file_key = lc(shift(@_));
310
311 # Check that the necessary arguments were supplied
312 if (!$target_language_code || !$translation_file_key) {
313 &throw_fatal_error("Missing command argument.");
314 }
315
316 # Get (and check) the translation configuration
317 my ($source_file, $target_file, $translation_file_type)
318 = &get_translation_configuration($target_language_code, $translation_file_key);
319
320 # Parse the source language and target language files
321 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
322 my @source_file_lines = &read_file_lines($source_file_path);
323 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
324
325 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
326 my @target_file_lines = &read_file_lines($target_file_path);
327 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
328
329 # Filter out any automatically translated chunks
330 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
331 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
332 delete $source_file_key_to_line_mapping{$chunk_key};
333 delete $target_file_key_to_line_mapping{$chunk_key};
334 }
335 }
336
337 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
338 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
339 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
340 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
341
342 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
343 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
344
345
346 # Chunks needing updating are those in the target file that have been more recently edited in the source file
347 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
348 my @uptodate_target_file_keys = ();
349 foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
350 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
351 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
352
353 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
354
355 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
356 # &log_message("Chunk with key $chunk_key needs updating.");
357 push(@uptodate_target_file_keys, $chunk_key);
358 }
359 }
360
361 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
362
363 return $xml_response;
364}
365
366
367sub get_first_n_chunks_requiring_work
368{
369 # The code of the target language (ensure it is lowercase)
370 my $target_language_code = lc(shift(@_));
371 # The key of the file to translate (ensure it is lowercase)
372 my $translation_file_key = lc(shift(@_));
373 # The number of chunks to return (defaults to one if not specified)
374 my $num_chunks_to_return = shift(@_) || "1";
375
376 # Check that the necessary arguments were supplied
377 if (!$target_language_code || !$translation_file_key) {
378 &throw_fatal_error("Missing command argument.");
379 }
380
381 # Get (and check) the translation configuration
382 my ($source_file, $target_file, $translation_file_type)
383 = &get_translation_configuration($target_language_code, $translation_file_key);
384
385 # Parse the source language and target language files
386 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
387 my @source_file_lines = &read_file_lines($source_file_path);
388 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
389
390 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
391 my @target_file_lines = &read_file_lines($target_file_path);
392 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
393
394 # Filter out any automatically translated chunks
395 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
396 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
397 delete $source_file_key_to_line_mapping{$chunk_key};
398 delete $target_file_key_to_line_mapping{$chunk_key};
399 }
400 }
401
402 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
403 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
404 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
405 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
406
407 # Determine the target file chunks requiring translation
408 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
409 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
410
411 # Determine the target file chunks requiring updating
412 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
413 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
414 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
415 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
416
417 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
418
419 return $xml_response;
420}
421
422
423sub get_language_status
424{
425 # The code of the target language (ensure it is lowercase)
426 my $target_language_code = lc(shift(@_));
427
428 # Check that the necessary arguments were supplied
429 if (!$target_language_code) {
430 &throw_fatal_error("Missing command argument.");
431 }
432
433 # Form an XML response to the command
434 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
435 $xml_response .= "<GTIResponse>\n";
436 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
437
438 foreach my $translation_file (@$gti_translation_files) {
439 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
440 my $target_file_name = "";
441
442 if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg, gs3deccolcfg
443 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
444 &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
445
446 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
447 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
448
449 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
450 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
451 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
452 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
453 }
454 else {
455 # Get (and check) the translation configuration
456 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
457 $target_file_name = $target_file;
458
459 # Parse the source language and target language files
460 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
461 my @source_file_lines = &read_file_lines($source_file_path);
462 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
463
464 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
465 my @target_file_lines = &read_file_lines($target_file_path);
466 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
467
468 # Filter out any automatically translated chunks
469 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
470 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
471 delete $source_file_key_to_line_mapping{$chunk_key};
472 delete $target_file_key_to_line_mapping{$chunk_key};
473 }
474 }
475
476 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
477 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
478
479 # Determine the target file chunks requiring translation
480 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
481
482 # Determine the target file chunks requiring updating
483 my @target_file_keys_requiring_updating = ();
484 if (-e $target_file_path) {
485 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
486 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
487 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
488 }
489
490 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
491 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
492 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
493 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
494 }
495
496 &log_message("Status of " . $translation_file->{'key'});
497 &log_message("Number of source chunks: " . $num_source_chunks);
498 &log_message("Number of target chunks: " . $num_target_chunks);
499 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
500 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
501
502 $xml_response .= " <TranslationFile"
503 . " key=\"" . $translation_file->{'key'} . "\""
504 . " target_file_path=\"" . $target_file_name . "\""
505 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
506 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
507 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
508 }
509
510 $xml_response .= " </LanguageStatus>\n";
511
512 $xml_response .= "</GTIResponse>\n";
513 return $xml_response;
514}
515
516
517sub search_chunks
518{
519 # The code of the target language (ensure it is lowercase)
520 my $target_language_code = lc(shift(@_));
521 # The key of the file to translate (ensure it is lowercase)
522 my $translation_file_key = lc(shift(@_));
523 # The query string
524 my $query_string = join(' ', @_);
525
526 # Check that the necessary arguments were supplied
527 if (!$target_language_code || !$translation_file_key || !$query_string) {
528 &throw_fatal_error("Missing command argument.");
529 }
530
531 my ($source_file, $target_file, $translation_file_type) = ();
532 my %source_file_key_to_text_mapping = ();
533 my %target_file_key_to_text_mapping = ();
534
535
536 if ($translation_file_key !~ m/^gs3/) {
537 # Get (and check) the translation configuration
538 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
539
540 # Parse the source language and target language files
541 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
542 my @source_file_lines = &read_file_lines($source_file_path);
543 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
544
545 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
546 my @target_file_lines = &read_file_lines($target_file_path);
547 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
548
549 # Filter out any automatically translated chunks
550 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
551 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
552 delete $source_file_key_to_line_mapping{$chunk_key};
553 delete $target_file_key_to_line_mapping{$chunk_key};
554 }
555 }
556
557 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
558 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
559 }
560 else {
561 # Not needed in this case
562 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
563 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
564 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
565 }
566
567 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
568 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
569
570 # Determine the target file chunks matching the query
571 my @target_file_keys_matching_query = ();
572 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
573 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
574 #&log_message("**** Next chunk to compare search term $query_string to: $target_file_text.");
575 if ($target_file_text =~ /$query_string/i) {
576 # &log_message("Chunk with key $chunk_key matches query.");
577 push(@target_file_keys_matching_query, $chunk_key);
578 }
579 }
580
581 # Form an XML response to the command
582 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
583 $xml_response .= "<GTIResponse>\n";
584
585 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
586 foreach my $chunk_key (@target_file_keys_matching_query) {
587 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
588
589 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
590 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
591 $xml_response .= " </Chunk>\n";
592 }
593 $xml_response .= " </ChunksMatchingQuery>\n";
594
595 $xml_response .= "</GTIResponse>\n";
596 return $xml_response;
597}
598
599
600sub submit_translations
601{
602 # The code of the target language (ensure it is lowercase)
603 my $target_language_code = lc(shift(@_));
604 # The key of the file to translate (ensure it is lowercase)
605 my $translation_file_key = lc(shift(@_));
606 # The username of the translation submitter
607 my $submitter_username = shift(@_);
608 # Whether to submit a target chunk even if it hasn't changed
609 my $force_submission_flag = shift(@_);
610
611 # Check that the necessary arguments were supplied
612 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
613 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
614 die "\n";
615 }
616
617 my %source_file_key_to_text_mapping = ();
618 my %source_file_key_to_gti_comment_mapping = ();
619 my %target_file_key_to_text_mapping = ();
620 my %target_file_key_to_gti_comment_mapping = ();
621
622 my (@source_file_lines, @target_file_lines) = ();
623 my ($source_file, $target_file, $translation_file_type);
624
625
626 if ($translation_file_key !~ m/^gs3/) {
627 # Get (and check) the translation configuration
628 ($source_file, $target_file, $translation_file_type)
629 = &get_translation_configuration($target_language_code, $translation_file_key);
630
631 #&log_message("********************** START SOURCE ***********************************");
632
633 # Parse the source language and target language files
634 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
635 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
636 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
637 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
638
639 #&log_message("********************** START TARGET ***********************************");
640
641 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
642 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
643 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
644 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
645 }
646 else {
647 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
648 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
649 }
650 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
651 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
652
653 # Submission date
654 my $day = (localtime)[3];
655 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
656 my $year = (localtime)[5] + 1900;
657 my $submission_date = "$day-$month-$year";
658
659 open(SUBMISSION, "-");
660 my @submission_lines = <SUBMISSION>;
661 close(SUBMISSION);
662
663 # Remove any nasty carriage returns
664 # &log_message("Submission:");
665 foreach my $submission_line (@submission_lines) {
666 $submission_line =~ s/\r$//;
667 $submission_line =~ s/ +$// if ($translation_file_key eq "glihelp");
668 #&log_message(" $submission_line");
669 }
670
671 my %source_file_key_to_submission_mapping = ();
672 my %target_file_key_to_submission_mapping = ();
673 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
674 # Read source file part of submission
675 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
676 my $chunk_key = $1;
677
678 # Read the source file text
679 my $source_file_chunk_text = "";
680 $i++;
681 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
682 $source_file_chunk_text .= $submission_lines[$i];
683 $i++;
684 }
685 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
686 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
687
688 #&log_message("Source file key: |$chunk_key|");
689 #&log_message("Source file text: |$source_file_chunk_text|");
690 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
691 }
692
693 # Read target file part of submission
694 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
695 my $chunk_key = $1;
696
697 # Read the target file text
698 my $target_file_chunk_text = "";
699 $i++;
700 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
701 $target_file_chunk_text .= $submission_lines[$i];
702 $i++;
703 }
704 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
705 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
706
707 #&log_message("Target file key: |$chunk_key|");
708 #&log_message("Target file text: |$target_file_chunk_text|");
709 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
710 }
711 }
712
713 # -----------------------------------------
714 # Validate the translation submissions
715 # -----------------------------------------
716
717 # Check that the translations are valid
718 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
719
720 # Kathy introduced escaped colons ("\:") into chunk keys in properties files (greenstone3/metadata_names),
721 # but they're not escaped in the submitted XML versions, nor are they escaped in memory (in the $chunk_key)
722
723 # Make sure the submitted chunk still exists in the source file
724 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
725 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
726 delete $source_file_key_to_submission_mapping{$chunk_key};
727 delete $target_file_key_to_submission_mapping{$chunk_key};
728 next;
729 }
730 ## DEBUG
731 # if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
732 # &log_message("**** ORIG SRC: " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n");
733 # &log_message("**** SUBMIT SRC: " . &unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) . "\n");
734 # } else {
735 # &log_message("unmake-XML-safe versions of source chunk before and for submission match");
736 # }
737 ## END DEBUG
738
739 # Make sure the submitted source chunk matches the source file chunk
740 #if ($source_file_key_to_submission_mapping{$chunk_key} ne $source_file_key_to_text_mapping{$chunk_key}
741 # && $source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
742 if (trim($source_file_key_to_submission_mapping{$chunk_key}) ne trim($source_file_key_to_text_mapping{$chunk_key})
743 && trim($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe(trim($source_file_key_to_text_mapping{$chunk_key}))) {
744 ##if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
745 #&log_message("**** submission source:\n|$source_file_key_to_submission_mapping{$chunk_key}|\n");
746 #&log_message("**** unmake xml safe source:\n|" . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."|\n");
747
748 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
749 &log_message("Submission source: |$source_file_key_to_submission_mapping{$chunk_key}|");
750 &log_message(" Source text: |$source_file_key_to_text_mapping{$chunk_key}|");
751 delete $source_file_key_to_submission_mapping{$chunk_key};
752 delete $target_file_key_to_submission_mapping{$chunk_key};
753 next;
754 }
755 }
756
757 # Apply the submitted translations
758 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
759 # Only apply the submission if it is a change, unless -force_submission has been specified
760 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
761 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
762 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
763 }
764 }
765
766 if ($translation_file_key !~ m/^gs3/) {
767 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
768 } else {
769 eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
770 }
771}
772
773
774# https://perlmaven.com/trim
775sub trim
776{
777 my $s = shift(@_);
778 $s =~ s/^\s+|\s+$//g;
779 return $s
780}
781
782sub create_glihelp_zip_file
783{
784 my $target_language_code = shift(@_);
785 my $translation_file_key = "glihelp";
786
787 &log_message("Creating GLI Help zip file for $target_language_code");
788
789 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
790
791 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
792 my $oldclasspath = $classpath;
793 if ( ! -e $classpath) {
794 $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
795 }
796 if ( ! -e $classpath) {
797 &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
798 }
799
800
801 my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
802 my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
803 my $os = $^O;
804 my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
805 my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
806 $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
807
808 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
809 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
810
811 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
812 if ( ! -e $gen_many_html_xsl_filepath) {
813 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
814 }
815
816 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
817 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
818
819 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
820 $target_file_directory = $target_file_directory."/";
821
822 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
823
824 # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
825 my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
826 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
827
828 # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
829 my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
830 my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
831 &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
832
833 # The following file reading section is a candidate to use FileUtils::readUTF8File()
834 # in place of calling sysread() directly. But only if we can reason we'd be working with UTF8
835 # In gli/help/<newlang>/help.xml, replace all occurrences of
836 # <Text id="1">This text in en will be removed for new langcode</Text>
837 # with <!-- Missing translation: 1 -->
838 open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
839 my $help_xml_contents;
840 # Read in the entire contents of the file in one hit
841 sysread(FIN, $help_xml_contents, -s FIN);
842 close(FIN);
843
844 $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
845
846 open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
847 print FOUT $help_xml_contents;
848 close(FOUT);
849 }
850
851 my $perl_exec = &util::get_perl_exec();
852
853 my $java_exec = undef;
854 if (defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne "") {
855 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
856 }
857 elsif (defined($ENV{'JRE_HOME'}) && $ENV{'JRE_HOME'} ne "") {
858 $java_exec = &util::filename_cat($ENV{'JRE_HOME'}, "bin", "java");
859 }
860 else {
861 # Assume it is on the user's PATH
862 $java_exec = "java";
863 }
864
865 #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
866 my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
867 #&throw_fatal_error("RAN gti command: $cmd");
868 my $response = `$cmd`;
869
870 #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
871 $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
872 $response = `$cmd`;
873
874 # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
875 my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
876 if(!&FileUtils::directoryExists($tmpdir)) {
877 &FileUtils::makeDirectory($tmpdir);
878 }
879 #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
880 my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
881 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
882
883 $response = `$cmd`;
884}
885
886
887sub get_translation_configuration
888{
889 # Get the code of the target language
890 my $target_language_code = shift(@_);
891 # Get the key of the file to translate
892 my $translation_file_key = shift(@_);
893
894 # Read the translation data from the gti.cfg file
895 my ($source_file, $target_file, $translation_file_type) =
896 &get_translation_data_for($target_language_code, $translation_file_key);
897
898 # Check that the file to translate is defined in the gti.cfg file
899 if (!$source_file || !$target_file || !$translation_file_type) {
900 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
901 }
902
903 # Check that the source file exists
904 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
905 if (!-e $source_file_path) {
906 &throw_fatal_error("Source file $source_file_path does not exist.");
907 }
908
909 # Check that the source file is up to date
910 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
911 # unless ($translation_file_is_not_in_cvs) {
912 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
913 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
914 if ($source_file_cvs_status =~ /^C /) {
915 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
916 }
917 if ($source_file_cvs_status =~ /^M /) {
918 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
919 }
920 # }
921
922 return ($source_file, $target_file, $translation_file_type);
923}
924
925
926sub get_translation_data_for
927{
928 my ($target_language_code, $translation_file_key) = @_;
929
930 foreach my $translation_file (@$gti_translation_files) {
931 # If this isn't the correct translation file, move onto the next one
932 next if ($translation_file_key ne $translation_file->{'key'});
933
934 # Resolve the target language file
935 my $target_language_file = $translation_file->{'target_file'};
936 if ($target_language_file =~ /(\{.+\;.+\})/) {
937 my $unresolved_target_language_file_part = $1;
938
939 # Check for a special case for the target language code
940 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
941 my $resolved_target_language_file_part = $2;
942 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
943 }
944 # Otherwise use the last part as the default value
945 else {
946 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
947 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
948 }
949 }
950
951 # Resolve instances of {iso_639_1_target_language_name}
952 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
953 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
954 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
955
956 # Resolve instances of {target_language_code}
957 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
958
959 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
960}
961
962return ();
963}
964
965
966sub read_file_lines
967{
968 my ($file_path) = @_;
969
970 if (!open(FILE_IN, "<$file_path")) {
971 &log_message("Note: Could not open file $file_path.");
972 return ();
973 }
974 my @file_lines = <FILE_IN>;
975 close(FILE_IN);
976
977 return @file_lines;
978}
979
980
981sub build_key_to_line_mapping
982{
983 my ($file_lines, $translation_file_type) = @_;
984 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
985}
986
987
988sub build_key_to_text_mapping
989{
990 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
991
992 my %key_to_text_mapping = ();
993 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
994 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
995 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
996
997 my $chunk_text = @$file_lines[$chunk_starting_line];
998 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
999 $chunk_text .= @$file_lines[$l];
1000 }
1001
1002 # Map from chunk key to text
1003 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
1004
1005 #&log_message("@@@ chunk key: $chunk_key");
1006 #&log_message("@@@ source string: |" . $key_to_text_mapping{$chunk_key} . "|");
1007
1008
1009 #if($chunk_key =~ m/document\\/) {
1010 #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
1011 #&log_message("@@@ chunk key: $chunk_key");
1012 #}
1013
1014 }
1015
1016 return %key_to_text_mapping;
1017}
1018
1019
1020sub build_key_to_last_update_date_mapping
1021{
1022 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1023
1024 # If the files aren't in CVS then we can't tell anything about what needs updating
1025 # return () if ($translation_file_is_not_in_cvs);
1026
1027 # Build a mapping from key to CVS date
1028 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
1029 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
1030
1031 # Build a mapping from key to comment date
1032 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
1033
1034 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
1035 my %key_to_last_update_date_mapping = ();
1036 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1037 # Use the CVS date as a starting point
1038 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
1039 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
1040
1041 # If a comment date exists and it is after the CVS date, use that instead
1042 # need to convert the comment date format to SVN format
1043 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
1044 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
1045 my $chunk_comment_date = $1;
1046 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
1047 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
1048 }
1049 }
1050 }
1051
1052 return %key_to_last_update_date_mapping;
1053}
1054
1055
1056sub build_key_to_cvs_date_mapping
1057{
1058 my ($filename, $translation_file_type) = @_;
1059
1060 # Use SVN to annotate each line of the file with the date it was last edited
1061 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
1062 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
1063
1064 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
1065
1066 my @cvs_annotated_file_lines_date = ();
1067 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
1068 # Extract the date from the SVN annotation at the front
1069 # svn format : 2007-07-16
1070 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
1071
1072 push(@cvs_annotated_file_lines_date, $1);
1073
1074 # trim extra date information in svn annotation format
1075 # 15:42:49 +1200 (Wed, 21 Jun 2006)
1076 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1077 }
1078
1079 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1080 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1081
1082 my %key_to_cvs_date_mapping = ();
1083 foreach my $chunk_key (keys(%key_to_line_mapping)) {
1084 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1085 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1086
1087 # Find the date this chunk was last edited, from the CVS annotation
1088 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
1089 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1090 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1091 # This part of the chunk has been updated more recently
1092 $chunk_date = $cvs_annotated_file_lines_date[$l];
1093
1094 }
1095 }
1096
1097 # Map from chunk key to CVS date
1098 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1099 }
1100
1101 return %key_to_cvs_date_mapping;
1102}
1103
1104
1105sub build_key_to_gti_comment_mapping
1106{
1107 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1108
1109 my %key_to_gti_comment_mapping = ();
1110 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1111 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1112 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1113
1114 my $chunk_text = @$file_lines[$chunk_starting_line];
1115 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1116 $chunk_text .= @$file_lines[$l];
1117 }
1118
1119 # Map from chunk key to GTI comment
1120 my $chunk_gti_comment;
1121 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1122 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1123 }
1124
1125 return %key_to_gti_comment_mapping;
1126}
1127
1128
1129sub determine_chunks_requiring_translation
1130{
1131 my $source_file_key_to_text_mapping = shift(@_);
1132 my $target_file_key_to_text_mapping = shift(@_);
1133
1134 # Chunks needing translation are those in the source file with no translation in the target file
1135 my @target_file_keys_requiring_translation = ();
1136 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1137 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1138 # &log_message("Chunk with key $chunk_key needs translating.");
1139 push(@target_file_keys_requiring_translation, $chunk_key);
1140 }
1141 }
1142
1143 return @target_file_keys_requiring_translation;
1144}
1145
1146
1147sub determine_chunks_requiring_updating
1148{
1149 my $source_file_key_to_last_update_date_mapping = shift(@_);
1150 my $target_file_key_to_last_update_date_mapping = shift(@_);
1151
1152 # Chunks needing updating are those in the target file that have been more recently edited in the source file
1153 my @target_file_keys_requiring_updating = ();
1154 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1155 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1156 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1157
1158 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1159
1160 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1161 # &log_message("Chunk with key $chunk_key needs updating.");
1162 # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1163 push(@target_file_keys_requiring_updating, $chunk_key);
1164 }
1165 }
1166
1167 return @target_file_keys_requiring_updating;
1168}
1169
1170
1171sub is_chunk_automatically_translated
1172{
1173 my ($chunk_key, $translation_file_type) = @_;
1174 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1175}
1176
1177
1178sub make_text_xml_safe
1179{
1180 my $text = shift(@_);
1181 $text =~ s/\&/\&amp\;/g;
1182 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1183 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1184 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1185 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1186 $text =~ s/</\&lt\;/g;
1187 $text =~ s/>/\&gt\;/g;
1188 return $text;
1189}
1190
1191
1192sub unmake_text_xml_safe
1193{
1194 my $text = shift(@_);
1195 $text =~ s/\&lt\;/</g;
1196 $text =~ s/\&gt\;/>/g;
1197 $text =~ s/\&amp\;/\&/g;
1198 return $text;
1199}
1200
1201
1202# Returns 1 if $date1 is after $date2, 0 otherwise
1203sub is_date_after_cvs
1204{
1205 my ($date1, $date2) = @_;
1206 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
1207 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1208
1209 if(!defined $date1) {
1210 return 1;
1211 }
1212
1213 my @date1parts = split(/-/, $date1);
1214 my @date2parts = split(/-/, $date2);
1215
1216 # Compare year - nasty because we have rolled over into a new century
1217 my $year1 = $date1parts[2];
1218 if ($year1 < 80) {
1219 $year1 += 2000;
1220 }
1221 my $year2 = $date2parts[2];
1222 if ($year2 < 80) {
1223 $year2 += 2000;
1224 }
1225
1226 # Compare year
1227 if ($year1 > $year2) {
1228 return 1;
1229 }
1230 elsif ($year1 == $year2) {
1231 # Year is the same, so compare month
1232 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1233 return 1;
1234 }
1235 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1236 # Month is the same, so compare day
1237 if ($date1parts[0] > $date2parts[0]) {
1238 return 1;
1239 }
1240 }
1241 }
1242
1243 return 0;
1244}
1245
1246sub is_date_after
1247{
1248 my ($date1, $date2) = @_;
1249
1250 if(!defined $date1) {
1251 return 1;
1252 }
1253 if(!defined $date2) {
1254 return 0;
1255 }
1256
1257 # 16-Aug-2006
1258 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1259 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1260 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1261 $date1=$3 . "-" . $months{$2} . "-" . $1;
1262 # print "** converted date1: $date1\n";
1263 }
1264 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1265 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1266 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1267 $date2=$3 . "-" . $months{$2} . "-" . $1;
1268 # print "** converted date2: $date2\n";
1269 }
1270
1271
1272 # 2006-08-16
1273 my @date1parts = split(/-/, $date1);
1274 my @date2parts = split(/-/, $date2);
1275
1276 # Compare year
1277 if ($date1parts[0] > $date2parts[0]) {
1278 return 1;
1279 }
1280 elsif ($date1parts[0] == $date2parts[0]) {
1281 # Year is the same, so compare month
1282 if ($date1parts[1] > $date2parts[1]) {
1283 return 1;
1284 }
1285 elsif ($date1parts[1] == $date2parts[1]) {
1286 # Month is the same, so compare day
1287 if ($date1parts[2] > $date2parts[2]) {
1288 return 1;
1289 }
1290 }
1291 }
1292
1293 return 0;
1294}
1295
1296
1297sub create_xml_response_for_chunks_requiring_work
1298{
1299 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1300
1301 # Form an XML response to the command
1302 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1303 $xml_response .= "<GTIResponse>\n";
1304 $xml_response .= " <TranslationFile"
1305 . " key=\"" . $translation_file_key . "\""
1306 . " target_file_path=\"" . $target_file . "\""
1307 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1308 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1309 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1310
1311 # Do chunks requiring translation first
1312 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1313 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1314 }
1315 else {
1316 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1317 }
1318
1319 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1320 foreach my $chunk_key (@sorted_chunk_keys) {
1321 last if ($num_chunks_to_return == 0);
1322
1323 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1324 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1325
1326 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1327 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1328 $xml_response .= " <TargetFileText></TargetFileText>\n";
1329 $xml_response .= " </Chunk>\n";
1330
1331 $num_chunks_to_return--;
1332 }
1333
1334 $xml_response .= " </ChunksRequiringTranslation>\n";
1335
1336 # Then do chunks requiring updating
1337 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1338 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1339 }
1340 else {
1341 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1342 }
1343
1344 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1345 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1346 foreach my $chunk_key (@sorted_chunk_keys) {
1347 last if ($num_chunks_to_return == 0);
1348
1349 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1350 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1351 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1352 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1353
1354 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1355 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1356 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1357 $xml_response .= " </Chunk>\n";
1358
1359 $num_chunks_to_return--;
1360 }
1361
1362 $xml_response .= " </ChunksRequiringUpdating>\n";
1363
1364 $xml_response .= "</GTIResponse>\n";
1365
1366 return $xml_response;
1367}
1368
1369sub create_xml_response_for_uptodate_chunks
1370{
1371 my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1372
1373 # Form an XML response to the command
1374 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1375 $xml_response .= "<GTIResponse>\n";
1376 $xml_response .= " <TranslationFile"
1377 . " key=\"" . $translation_file_key . "\""
1378 . " target_file_path=\"" . $target_file . "\""
1379 . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1380
1381
1382 # Then do chunks requiring updating
1383 $xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1384
1385
1386 # foreach my $chunk_key (@uptodate_target_file_keys) {
1387 my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1388 foreach my $chunk_key (@sorted_chunk_keys) {
1389
1390 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1391 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1392 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1393 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1394
1395 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1396 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1397 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1398 $xml_response .= " </Chunk>\n";
1399
1400 }
1401
1402 $xml_response .= " </UptodateChunks>\n";
1403
1404 $xml_response .= "</GTIResponse>\n";
1405
1406 return $xml_response;
1407}
1408
1409sub create_xml_response_for_all_chunks
1410{
1411 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1412
1413 # Form an XML response to the command
1414 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1415 $xml_response .= "<GTIResponse>\n";
1416 $xml_response .= " <TranslationFile"
1417 . " key=\"" . $translation_file_key . "\""
1418 . " target_file_path=\"" . $target_file . "\"\/>\n";
1419
1420 # Do all the chunks
1421 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1422
1423 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1424 foreach my $chunk_key (@sorted_chunk_keys) {
1425 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1426 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1427
1428 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1429 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1430 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1431 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1432 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1433 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1434 }
1435 else {
1436 $xml_response .= " <TargetFileText></TargetFileText>\n";
1437 }
1438
1439 $xml_response .= " </Chunk>\n";
1440 }
1441 $xml_response .= " </Chunks>\n";
1442
1443 $xml_response .= "</GTIResponse>\n";
1444 return $xml_response;
1445}
1446
1447
1448
1449# ==========================================================================================
1450# MACROFILE FUNCTIONS
1451
1452sub build_key_to_line_mapping_for_macrofile
1453{
1454 my (@file_lines) = @_;
1455
1456 my $macro_package;
1457 my %chunk_key_to_line_mapping = ();
1458 # Process the contents of the file, line by line
1459 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1460 my $line = $file_lines[$i];
1461 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1462
1463 # Check if a new package is being defined
1464 if ($line =~ m/^package\s+(.+)/) {
1465 $macro_package = $1;
1466 }
1467
1468 # Line contains a macro name
1469 elsif ($line =~ m/^(_\w+_)/) {
1470 my $macro_key = $1;
1471 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1472
1473 # While there is still text of the macro to go...
1474 my $startline = $i;
1475 while ($line !~ /\}$/) {
1476 $i++;
1477 if ($i == scalar(@file_lines)) {
1478 &throw_fatal_error("Could not find end of macro $macro_key.");
1479 }
1480 $line = $file_lines[$i];
1481 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1482 }
1483
1484 # The chunk key consists of the package name and the macro key
1485 my $chunk_key = $macro_package . "." . $macro_key;
1486 # Map from chunk key to line
1487 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1488 }
1489
1490 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1491 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1492 # The chunk key consists of package name and macro key
1493 my $chunk_key = $macro_package . "." . $1;
1494 # Map from chunk key to line
1495 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1496}
1497}
1498
1499return %chunk_key_to_line_mapping;
1500}
1501
1502
1503sub import_chunk_from_macrofile
1504{
1505 my ($chunk_text) = @_;
1506
1507 # Is this an icon macro??
1508 if ($chunk_text =~ /^\#\# (.*)/) {
1509 # Extract image macro text
1510 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1511 $chunk_text = $1;
1512
1513 # Remove enclosing quotes
1514 $chunk_text =~ s/^\"//;
1515 $chunk_text =~ s/\"$//;
1516 }
1517
1518 # No, so it must be a text macro
1519 else {
1520 # Remove macro key
1521 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1522
1523 # Remove language specifier
1524 $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1525
1526 # Remove braces enclosing text
1527 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1528 }
1529
1530 return $chunk_text;
1531}
1532
1533
1534sub get_macrofile_chunk_gti_comment
1535{
1536 my ($chunk_text) = @_;
1537
1538 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1539 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1540 return $1;
1541}
1542
1543return undef;
1544}
1545
1546
1547sub is_macrofile_chunk_automatically_translated
1548{
1549 my ($chunk_key) = @_;
1550
1551 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1552 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1553 return 1;
1554 }
1555
1556 return 0;
1557}
1558
1559
1560# Use the source file to generate a target file that is formatted the same
1561sub write_translated_macrofile
1562{
1563 my $source_file = shift(@_); # Not used
1564 my @source_file_lines = @{shift(@_)};
1565 my $source_file_key_to_text_mapping = shift(@_);
1566 my $target_file = shift(@_);
1567 my @target_file_lines = @{shift(@_)};
1568 my $target_file_key_to_text_mapping = shift(@_);
1569 my $target_file_key_to_gti_comment_mapping = shift(@_);
1570 my $target_language_code = shift(@_);
1571
1572 # Build a mapping from source file line to chunk key
1573 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1574 my %source_file_line_to_key_mapping = ();
1575 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1576 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1577 }
1578 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1579 my $source_file_line_number = 0;
1580
1581 # Build a mapping from target file line to chunk key
1582 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1583 my %target_file_line_to_key_mapping = ();
1584 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1585 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1586 }
1587 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1588
1589 # Write the new target file
1590 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1591 if (!open(TARGET_FILE, ">$target_file_path")) {
1592 &throw_fatal_error("Could not write target file $target_file_path.");
1593 }
1594
1595 # Use the header from the target file, to keep language and author information
1596 if (scalar(@target_file_line_keys) > 0) {
1597 my $target_file_line_number = 0;
1598 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1599 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1600 my $target_file_line = $target_file_lines[$target_file_line_number];
1601 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1602 print TARGET_FILE $target_file_line;
1603 $target_file_line_number++;
1604 }
1605
1606 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1607 }
1608
1609 # Model the new target file on the source file, with the target file translations
1610 foreach my $line_key (@source_file_line_keys) {
1611 # Fill in the gaps before this chunk starts
1612 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1613 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1614 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1615 print TARGET_FILE $source_file_lines[$source_file_line_number];
1616 $source_file_line_number++;
1617 }
1618 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1619
1620 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1621 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1622 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1623
1624 my $macrofile_key = $chunk_key;
1625 $macrofile_key =~ s/^(.+?)\.//;
1626
1627 # If no translation exists for this chunk, show this, and move on
1628 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1629 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1630 next;
1631 }
1632
1633 # Grab the source chunk text
1634 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1635 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1636 $source_file_chunk .= $source_file_lines[$l];
1637 }
1638
1639 # Is this an icon macro??
1640 if ($source_file_chunk =~ /^\#\# (.*)/) {
1641 # Escape any newline and question mark characters so the source text is replaced correctly
1642 $source_file_chunk_text =~ s/\\/\\\\/g;
1643 $source_file_chunk_text =~ s/\?/\\\?/g;
1644
1645 # Build the new target chunk from the source chunk
1646 my $target_file_chunk = $source_file_chunk;
1647 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1648 $target_file_chunk =~ s/(\s)*$//;
1649 print TARGET_FILE "$target_file_chunk";
1650 }
1651
1652 # No, it is just a normal text macro
1653 else {
1654 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1655 }
1656
1657 # Add the "updated" comment, if one exists
1658 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1659 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1660 }
1661 print TARGET_FILE "\n";
1662}
1663
1664close(TARGET_FILE);
1665}
1666
1667
1668sub sort_by_line
1669{
1670 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1671}
1672
1673
1674# ==========================================================================================
1675# RESOURCE BUNDLE FUNCTIONS
1676
1677# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1678sub build_key_to_line_mapping_for_resource_bundle
1679{
1680 my (@file_lines) = @_;
1681
1682 my %chunk_key_to_line_mapping = ();
1683
1684 my $chunk_key;
1685 my $startindex = -1;
1686
1687 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1688 my $line = $file_lines[$i];
1689 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1690 #&log_message("Got line: ".$line);
1691
1692 # a property line has a colon/equals sign as separator that is NOT escaped with a backslash (both keys and values
1693 # can use the colon or = sign. But in the key, such a char is always escaped. Unfortunately, they've not always been
1694 # escaped in the values. So we get the left most occurrence by not doing a greedy match (use ? to not be greedy).
1695 # So find the first :/= char not preceded by \. That will be the true separator of a chunk_key and its value chunk_text
1696
1697 if ($line =~ m/^(\S*?[^\\])[:|=](.*)$/) {
1698 # Line contains a dictionary string
1699
1700 # Unused but useful: http://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
1701 # http://perldoc.perl.org/perlvar.html
1702
1703 $chunk_key = $1;
1704 # remove the escaping of any :/= property separator from the chunk_key in memory,
1705 # to make comparison with its unescaped version during submissions easier. Will write out with escaping.
1706 $chunk_key =~ s/\\([:=])/$1/g;
1707
1708 $startindex = $i;
1709 }
1710 if ($startindex != -1) {
1711 if($line !~ m/\\$/) { # line finished
1712 # $i keeps track of the line at which this property (chunk_key) finishes
1713
1714 # Map from chunk key to line
1715 $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1716 $startindex = -1;
1717 $chunk_key = "";
1718 }
1719 }
1720 }
1721
1722 return %chunk_key_to_line_mapping;
1723}
1724
1725
1726sub import_chunk_from_resource_bundle
1727{
1728 my ($chunk_text) = @_;
1729
1730 # Simple: just remove string key.
1731 # But key can contain an escaped separator (\: or \=).
1732 # So just as in the previous subroutine, find the first (leftmost) : or = char not preceded by \.
1733 # That will be the true separator of a chunk_key and its value chunk_text
1734 $chunk_text =~ s/^(\S*?[^\\])[:|=](\s*)//s;
1735
1736 $chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
1737 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1738
1739 return $chunk_text;
1740}
1741
1742
1743sub get_resource_bundle_chunk_gti_comment
1744{
1745 my ($chunk_text) = @_;
1746
1747 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1748 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1749 return $1;
1750 }
1751
1752 return undef;
1753}
1754
1755
1756sub is_resource_bundle_chunk_automatically_translated
1757{
1758 # No resource bundle chunks are automatically translated
1759 return 0;
1760}
1761
1762
1763sub write_translated_resource_bundle
1764{
1765 my $source_file = shift(@_); # Not used
1766 my @source_file_lines = @{shift(@_)};
1767 my $source_file_key_to_text_mapping = shift(@_);
1768 my $target_file = shift(@_);
1769 my @target_file_lines = @{shift(@_)}; # Not used
1770 my $target_file_key_to_text_mapping = shift(@_);
1771 my $target_file_key_to_gti_comment_mapping = shift(@_);
1772 my $target_language_code = shift(@_); # Not used
1773
1774 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1775 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1776 my %source_file_line_to_key_mapping = ();
1777 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1778 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1779 }
1780
1781 # Write the new target file
1782 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1783 if (!open(TARGET_FILE, ">$target_file_path")) {
1784 &throw_fatal_error("Could not write target file $target_file_path.");
1785 }
1786
1787 # Model the new target file on the source file, with the target file translations
1788 my $source_file_line_number = 0;
1789 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1790 # Fill in the gaps before this chunk starts
1791 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1792 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1793 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1794 print TARGET_FILE $source_file_lines[$source_file_line_number];
1795 $source_file_line_number++;
1796 }
1797 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1798
1799 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1800 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1801 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1802
1803 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
1804 # since the key-value separator in a property resource bundle file is : or =
1805 my $escaped_chunk_key = $chunk_key;
1806 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
1807
1808 # If no translation exists for this chunk, show this, and move on
1809 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1810 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
1811 next;
1812 }
1813
1814 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
1815 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1816 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1817 }
1818 print TARGET_FILE "\n";
1819 }
1820
1821 close(TARGET_FILE);
1822}
1823
1824
1825# ==========================================================================================
1826# GREENSTONE XML FUNCTIONS
1827
1828sub build_key_to_line_mapping_for_greenstone_xml
1829{
1830 my (@file_lines) = @_;
1831
1832 my %chunk_key_to_line_mapping = ();
1833 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1834 my $line = $file_lines[$i];
1835 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1836 #&log_message("Got line: ".$line);
1837 # Line contains a string to translate
1838 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1839 my $chunk_key = $1;
1840 $line =~ s/\s*$//; # Remove any nasty whitespace
1841 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1842
1843 # While there is still text of the string to go...
1844 my $startline = $i;
1845 while ($line !~ /<\/Text>\s*$/) {
1846 $i++;
1847 if ($i == scalar(@file_lines)) {
1848 &throw_fatal_error("Could not find end of string $chunk_key.");
1849 }
1850 $line = $file_lines[$i];
1851 $line =~ s/\s*$//; # Remove any nasty whitespace
1852 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1853 }
1854 #&log_message("@@@ Line is now: ".$line);
1855 # Map from chunk key to line
1856 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1857 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1858 }
1859 else {
1860 &throw_fatal_error("Duplicate key $chunk_key.");
1861 }
1862 }
1863 }
1864
1865 return %chunk_key_to_line_mapping;
1866}
1867
1868
1869sub import_chunk_from_greenstone_xml
1870{
1871 my ($chunk_text) = @_;
1872
1873 # Simple: just remove the Text tags
1874 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1875 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1876 $chunk_text =~ s/\s*<\/Text>\s*$//;
1877
1878 return $chunk_text;
1879}
1880
1881
1882sub get_greenstone_xml_chunk_gti_comment
1883{
1884 my ($chunk_text) = @_;
1885
1886 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1887 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1888 return $1;
1889 }
1890
1891 return undef;
1892}
1893
1894
1895sub is_greenstone_xml_chunk_automatically_translated
1896{
1897 # No greenstone XML chunks are automatically translated
1898 return 0;
1899}
1900
1901
1902sub write_translated_greenstone_xml
1903{
1904 my $source_file = shift(@_); # Not used
1905 my @source_file_lines = @{shift(@_)};
1906 my $source_file_key_to_text_mapping = shift(@_);
1907 my $target_file = shift(@_);
1908 my @target_file_lines = @{shift(@_)}; # Not used
1909 my $target_file_key_to_text_mapping = shift(@_);
1910 my $target_file_key_to_gti_comment_mapping = shift(@_);
1911 my $target_language_code = shift(@_); # Not used
1912
1913 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1914 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1915 my %source_file_line_to_key_mapping = ();
1916 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1917 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1918 }
1919
1920 # Write the new target file
1921 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1922 if (!open(TARGET_FILE, ">$target_file_path")) {
1923 &throw_fatal_error("Could not write target file $target_file_path.");
1924 }
1925
1926 # Model the new target file on the source file, with the target file translations
1927 my $source_file_line_number = 0;
1928 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1929 # Fill in the gaps before this chunk starts
1930 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1931 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1932 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1933 print TARGET_FILE $source_file_lines[$source_file_line_number];
1934 $source_file_line_number++;
1935 }
1936 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1937
1938 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1939 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1940 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1941 $target_file_chunk_text =~ s/(\n)*$//g;
1942
1943 # If no translation exists for this chunk, show this, and move on
1944 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1945 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1946 next;
1947 }
1948
1949 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1950 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1951 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1952 $chunk_gti_comment =~ s/^Updated //;
1953 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1954 }
1955 print TARGET_FILE "\n";
1956 }
1957
1958 # Fill in the end of the file
1959 while ($source_file_line_number < scalar(@source_file_lines)) {
1960 print TARGET_FILE $source_file_lines[$source_file_line_number];
1961 $source_file_line_number++;
1962 }
1963
1964 close(TARGET_FILE);
1965}
1966
1967
1968# ==========================================================================================
1969# GREENSTONE3 FUNCTIONS
1970
1971sub get_all_chunks_for_gs3
1972{
1973 # The code of the target language (ensure it is lowercase)
1974 my $target_language_code = lc(shift(@_));
1975 my $translation_file_key = lc(shift(@_));
1976
1977 # Check that the necessary arguments were supplied
1978 if (!$target_language_code) {
1979 &throw_fatal_error("Missing command argument.");
1980 }
1981
1982 # Get (and check) the translation configuration
1983 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1984
1985 my %source_files_key_to_text_mapping = ();
1986 my %target_files_key_to_text_mapping = ();
1987 my %source_files_key_to_last_update_date_mapping = ();
1988 my %target_files_key_to_last_update_date_mapping = ();
1989
1990 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1991
1992 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1993 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1994
1995 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1996 return $xml_response;
1997}
1998
1999
2000sub get_first_n_chunks_requiring_work_for_gs3
2001{
2002 # The code of the target language (ensure it is lowercase)
2003 my $target_language_code = lc(shift(@_));
2004 # The key of the file to translate (ensure it is lowercase)
2005 my $translation_file_key = lc(shift(@_));
2006 # The number of chunks to return (defaults to one if not specified)
2007 my $num_chunks_to_return = shift(@_) || "1";
2008
2009 # Check that the necessary arguments were supplied
2010 if (!$target_language_code || !$translation_file_key) {
2011 &throw_fatal_error("Missing command argument.");
2012 }
2013
2014 my %source_files_key_to_text_mapping = ();
2015 my %target_files_key_to_text_mapping = ();
2016 my %source_files_key_to_last_update_date_mapping = ();
2017 my %target_files_key_to_last_update_date_mapping = ();
2018
2019 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2020 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2021
2022 # Determine the target file chunks requiring translation
2023 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
2024 # Determine the target file chunks requiring updating
2025 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2026 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
2027 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
2028
2029 my $download_target_filepath = "";
2030
2031
2032 # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
2033
2034 # if there is no copy of the language files for download, there's also no link to the spreadsheet
2035 # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
2036 # interface *.properties files,and link to that zip as the file for offline translation.
2037 # Selecting only properties files for English and the language they're working on (if the last exists)
2038
2039 # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
2040 # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
2041
2042 my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
2043 my $tar_cmd = "tar -cvzf $zip";
2044
2045
2046 # store cur dir and cd to gsdlhome to generate the correct path in the zip file
2047 my $curdir = `pwd`;
2048 chdir $gsdl_root_directory;
2049
2050 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
2051 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
2052 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-dec-col-cfgs", \@gs3_dec_col_cfg_files);
2053
2054 # tar command will overwrite the previous version, but want to check we've created it
2055 if(&FileUtils::fileExists($zip)) {
2056 &FileUtils::removeFiles($zip);
2057 }
2058
2059 #my $tar_result = system($tar_cmd); # works but then interface breaks
2060 `$tar_cmd`;
2061 my $tar_result = $?;
2062
2063 if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
2064 $download_target_filepath = $zip;
2065 } else {
2066 &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
2067 }
2068
2069 # change back to original working directory (cgi-bin/linux probably)
2070 chdir $curdir;
2071
2072 # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
2073
2074
2075 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
2076 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
2077 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2078 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2079
2080 return $xml_response;
2081}
2082
2083# helper function
2084# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg, gs3deccolccfg)
2085# formatted correctly to go into a zip file
2086sub get_gs3_zip_file_listing
2087{
2088 my $target_language_code = shift(@_);
2089 my $sourcedir = shift(@_);
2090 my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
2091
2092 my $filelisting = "";
2093 foreach my $interface_file (@$files_array) {
2094
2095 my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
2096 my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2097
2098 $filelisting = "$filelisting $source_filepath";
2099 if(&FileUtils::fileExists($target_filepath)) {
2100 $filelisting = "$filelisting $target_filepath";
2101 }
2102 }
2103
2104 return $filelisting;
2105}
2106
2107sub get_uptodate_chunks_for_gs3
2108{
2109 # The code of the target language (ensure it is lowercase)
2110 my $target_language_code = lc(shift(@_));
2111 # The key of the file to translate (ensure it is lowercase)
2112 my $translation_file_key = lc(shift(@_));
2113 # The number of chunks to return (defaults to one if not specified)
2114 my $num_chunks_to_return = shift(@_) || "1";
2115
2116 # Check that the necessary arguments were supplied
2117 if (!$target_language_code || !$translation_file_key) {
2118 &throw_fatal_error("Missing command argument.");
2119 }
2120
2121 my %source_files_key_to_text_mapping = ();
2122 my %target_files_key_to_text_mapping = ();
2123 my %source_files_key_to_last_update_date_mapping = ();
2124 my %target_files_key_to_last_update_date_mapping = ();
2125
2126 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2127 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2128
2129
2130 # Chunks needing updating are those in the target file that have been more recently edited in the source file
2131 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2132 my @uptodate_target_file_keys = ();
2133 foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2134 my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2135 my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2136
2137 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
2138
2139 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2140 # &log_message("Chunk with key $chunk_key needs updating.");
2141 push(@uptodate_target_file_keys, $chunk_key);
2142 }
2143 }
2144
2145 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2146
2147 return $xml_response;
2148}
2149
2150
2151sub build_gs3_configuration
2152{
2153 my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2154 $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2155
2156 my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2157 my $files_array = \@gs3_interface_files;
2158
2159 if($translation_file_key eq "gs3colcfg") {
2160 $source_file_directory = "gs3-collection-configs";
2161 $files_array = \@gs3_col_cfg_files;
2162 } elsif($translation_file_key eq "gs3deccolcfg") {
2163 $source_file_directory = "gs3-dec-col-cfgs";
2164 $files_array = \@gs3_dec_col_cfg_files;
2165 }
2166 my $translation_file_type = "resource_bundle";
2167
2168 foreach my $interface_file_key (@$files_array) {
2169
2170 &log_message("Greenstone 3 interface file: " . $interface_file_key);
2171
2172 # Parse the source language and target language files
2173 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2174 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2175 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2176 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2177 #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2178
2179 my %source_file_key_to_gti_comment_or_last_updated_mapping;
2180 if($get_gti_comments_not_last_updated) {
2181 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2182 } else {
2183 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2184 }
2185
2186 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2187 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2188 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2189 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2190 #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2191
2192 my %target_file_key_to_gti_comment_or_last_updated_mapping;
2193 if($get_gti_comments_not_last_updated) {
2194 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2195 } else {
2196 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2197 }
2198
2199
2200 # Filter out any automatically translated chunks
2201 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2202 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2203 delete $source_file_key_to_line_mapping{$chunk_key};
2204 delete $target_file_key_to_line_mapping{$chunk_key};
2205 }
2206 }
2207
2208 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2209 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2210
2211 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2212 my $global_chunk_key = "$interface_file_key.$chunk_key";
2213 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2214 $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2215
2216 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2217 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2218 $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2219 }
2220 }
2221 }
2222}
2223
2224
2225sub write_translated_gs3interface
2226{
2227 my $translation_file_key = shift(@_);
2228 my $source_file_key_to_text_mapping = shift(@_);
2229 my $target_file_key_to_text_mapping = shift(@_);
2230 my $target_file_key_to_gti_comment_mapping = shift(@_);
2231 my $target_language_code = shift(@_);
2232
2233 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2234
2235 my %translated_interface_file_keys = ();
2236 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2237 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2238 if (!defined $translated_interface_file_keys{$1}) {
2239 &log_message("Updated interface file: " . $1);
2240 $translated_interface_file_keys{$1}="";
2241 }
2242 }
2243 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2244
2245 my $source_file_directory = "greenstone3";
2246 $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2247 $source_file_directory = "gs3-dec-col-cfgs" if $translation_file_key eq "gs3deccolcfg";
2248
2249 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2250
2251 # Build a mapping from chunk key to source file line, and from source file line to chunk key
2252 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2253 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2254 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2255 my %source_file_line_to_key_mapping = ();
2256 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2257 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2258 }
2259
2260 # Write the new target file
2261 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2262 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2263 if (!open(TARGET_FILE, ">$target_file_path")) {
2264 &throw_fatal_error("Could not write target file $target_file_path.");
2265 }
2266
2267 # Model the new target file on the source file, with the target file translations
2268 my $source_file_line_number = 0;
2269 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2270 # Fill in the gaps before this chunk starts
2271 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2272 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2273 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2274 print TARGET_FILE $source_file_lines[$source_file_line_number];
2275 $source_file_line_number++;
2276 }
2277 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2278
2279 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2280 my $global_chunk_key = "$interface_file_key.$chunk_key";
2281 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2282 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2283
2284 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
2285 # since the key-value separator in a property resource bundle file is : or =
2286 my $escaped_chunk_key = $chunk_key;
2287 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
2288
2289 # If no translation exists for this chunk, show this, and move on
2290 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2291 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
2292 next;
2293 }
2294
2295 if ($translation_file_key eq "gs3deccolcfg") { # leave in the equals sign as key-value separator in gs3deccolcfg module
2296 print TARGET_FILE "$escaped_chunk_key=$target_file_chunk_text";
2297 } else { # for other modules, do as before: use a colon as key-value separator
2298 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
2299 }
2300 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2301 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2302 }
2303 print TARGET_FILE "\n";
2304 }
2305
2306 close(TARGET_FILE);
2307 }
2308}
2309
2310&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.