source: main/trunk/greenstone2/bin/script/gti.pl@ 32517

Last change on this file since 32517 was 32517, checked in by ak19, 3 years ago

Fixing a bug that Kathy found: serbian (bosnian herzegovina) cyrillic and latin variants (sr-bh-cyr and sr-bh-lat) were not retrieving all the existing translated strings when viewing the status page. The macros coredm and auxdm modules were certainly not finding the correct translation files to get info on the number of strings already translated in those modules. This has now been fixed.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 96.1 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en"; # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48 'file_type' => "macrofile",
49 'source_file' => "macros/english.dm",
50 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;sr-bh-cyr:serbian-bh-cyr;sr-bh-lat:serbian-bh-lat;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53 'file_type' => "macrofile",
54 'source_file' => "macros/english2.dm",
55 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58# 'file_type' => "macrofile",
59# 'source_file' => "macros/paperspast-english.dm",
60# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64 'file_type' => "resource_bundle",
65 'source_file' => "gli/classes/dictionary.properties",
66 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70 'file_type' => "greenstone_xml",
71 'source_file' => "gli/help/en/help.xml",
72 'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76 'file_type' => "resource_bundle",
77 'source_file' => "perllib/strings.properties",
78 'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82 'file_type' => "resource_bundle",
83 'source_file' => "gsinstaller/LanguagePack.properties",
84 'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94 'file_type' => "resource_bundle",
95 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96 'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102 'file_type' => "resource_bundle",
103 'source_file' => "greenstone3",
104 'target_file' => "greenstone3"
105},
106
107# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
108# from http://svn.greenstone.org/main/trunk/gs3-collection-configs
109{ 'key' => "gs3colcfg",
110 'file_type' => "resource_bundle",
111 'source_file' => "gs3-collection-configs",
112 'target_file' => "gs3-collection-configs"
113}
114];
115
116my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo", "localsite");
117
118my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
119#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
120
121# Auxilliary GS3 interface files. This list is not used at present
122# Combine with above list if generating translation spreadsheet for all interface files
123my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
124
125my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
126
127# Not: i18n, log4j
128
129sub main
130{
131 # Get the command to process, and any arguments
132 my $gti_command = shift(@_);
133 my @gti_command_arguments = @_;
134 my $module = $_[1];
135
136 # for GS3, set gsdl_root_dir to GSDL3HOME
137 #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
138 #if($ENV{'GSDL3SRCHOME'}) {
139 # $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
140 # $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
141 #}
142 #}
143
144 # Open the GTI log file for appending, or write to STDERR if that fails
145 if (!open(GTI_LOG, ">>$gti_log_file")) {
146 open(GTI_LOG, ">&STDERR");
147 }
148
149 # Log the command that launched this script
150 &log_message("Command: $0 @ARGV");
151
152 # Check that a command was supplied
153 if (!$gti_command) {
154 &throw_fatal_error("Missing command.");
155 }
156
157 # Process the command
158 if ($gti_command =~ /^get-all-chunks$/i) {
159 # Check that GS3 interface is the target
160 if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg
161 print &get_all_chunks_for_gs3(@gti_command_arguments);
162 } else {
163 print &get_all_chunks(@gti_command_arguments);
164 }
165 }
166 elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
167 if ($module =~ m/^gs3/) {
168 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
169 } else {
170 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
171 }
172 }
173 elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
174 if ($module =~ m/^gs3/) {
175 print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
176 } else {
177 print &get_uptodate_chunks(@gti_command_arguments);
178 }
179 }
180 elsif ($gti_command =~ /^get-language-status$/i) {
181 print &get_language_status(@gti_command_arguments);
182 }
183 elsif ($gti_command =~ /^search-chunks$/i) {
184 print &search_chunks(@gti_command_arguments);
185 }
186 elsif ($gti_command =~ /^submit-translations$/i) {
187 # This command cannot produce any output since it reads input
188 &submit_translations(@gti_command_arguments);
189 }
190 elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
191 # This command cannot produce any output since it reads input
192 &create_glihelp_zip_file(@gti_command_arguments);
193 }
194 else {
195 # The command was not recognized
196 &throw_fatal_error("Unknown command \"$gti_command\".");
197 }
198}
199
200
201sub throw_fatal_error
202{
203 my $error_message = shift(@_);
204
205 # Write an XML error response
206 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
207 print "<GTIResponse>\n";
208 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
209 print "</GTIResponse>\n";
210
211 # Log the error message, then die
212 &log_message("Error: $error_message");
213 die "\n";
214}
215
216
217sub log_message
218{
219 my $log_message = shift(@_);
220 print GTI_LOG time() . " -- " . $log_message . "\n";
221}
222
223
224sub get_all_chunks
225{
226 # The code of the target language (ensure it is lowercase)
227 my $target_language_code = lc(shift(@_));
228 # The key of the file to translate (ensure it is lowercase)
229 my $translation_file_key = lc(shift(@_));
230
231 # Check that the necessary arguments were supplied
232 if (!$target_language_code || !$translation_file_key) {
233 &throw_fatal_error("Missing command argument.");
234 }
235
236 # Get (and check) the translation configuration
237 my ($source_file, $target_file, $translation_file_type)
238 = &get_translation_configuration($target_language_code, $translation_file_key);
239
240 # Parse the source language and target language files
241 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
242 my @source_file_lines = &read_file_lines($source_file_path);
243 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
244
245 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
246 my @target_file_lines = &read_file_lines($target_file_path);
247 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
248
249 # Filter out any automatically translated chunks
250 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
251 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
252 delete $source_file_key_to_line_mapping{$chunk_key};
253 delete $target_file_key_to_line_mapping{$chunk_key};
254 }
255 }
256
257 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
258 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
259 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
260 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
261
262 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
263 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
264
265 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
266
267 return $xml_response;
268}
269
270
271sub get_uptodate_chunks
272{
273 # The code of the target language (ensure it is lowercase)
274 my $target_language_code = lc(shift(@_));
275 # The key of the file to translate (ensure it is lowercase)
276 my $translation_file_key = lc(shift(@_));
277
278 # Check that the necessary arguments were supplied
279 if (!$target_language_code || !$translation_file_key) {
280 &throw_fatal_error("Missing command argument.");
281 }
282
283 # Get (and check) the translation configuration
284 my ($source_file, $target_file, $translation_file_type)
285 = &get_translation_configuration($target_language_code, $translation_file_key);
286
287 # Parse the source language and target language files
288 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
289 my @source_file_lines = &read_file_lines($source_file_path);
290 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
291
292 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
293 my @target_file_lines = &read_file_lines($target_file_path);
294 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
295
296 # Filter out any automatically translated chunks
297 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
298 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
299 delete $source_file_key_to_line_mapping{$chunk_key};
300 delete $target_file_key_to_line_mapping{$chunk_key};
301 }
302 }
303
304 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
305 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
306 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
307 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
308
309 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
310 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
311
312
313 # Chunks needing updating are those in the target file that have been more recently edited in the source file
314 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
315 my @uptodate_target_file_keys = ();
316 foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
317 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
318 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
319
320 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
321
322 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
323 # &log_message("Chunk with key $chunk_key needs updating.");
324 push(@uptodate_target_file_keys, $chunk_key);
325 }
326 }
327
328 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
329
330 return $xml_response;
331}
332
333
334sub get_first_n_chunks_requiring_work
335{
336 # The code of the target language (ensure it is lowercase)
337 my $target_language_code = lc(shift(@_));
338 # The key of the file to translate (ensure it is lowercase)
339 my $translation_file_key = lc(shift(@_));
340 # The number of chunks to return (defaults to one if not specified)
341 my $num_chunks_to_return = shift(@_) || "1";
342
343 # Check that the necessary arguments were supplied
344 if (!$target_language_code || !$translation_file_key) {
345 &throw_fatal_error("Missing command argument.");
346 }
347
348 # Get (and check) the translation configuration
349 my ($source_file, $target_file, $translation_file_type)
350 = &get_translation_configuration($target_language_code, $translation_file_key);
351
352 # Parse the source language and target language files
353 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
354 my @source_file_lines = &read_file_lines($source_file_path);
355 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
356
357 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
358 my @target_file_lines = &read_file_lines($target_file_path);
359 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
360
361 # Filter out any automatically translated chunks
362 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
363 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
364 delete $source_file_key_to_line_mapping{$chunk_key};
365 delete $target_file_key_to_line_mapping{$chunk_key};
366 }
367 }
368
369 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
372 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
373
374 # Determine the target file chunks requiring translation
375 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
376 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
377
378 # Determine the target file chunks requiring updating
379 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
380 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
381 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
382 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
383
384 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
385
386 return $xml_response;
387}
388
389
390sub get_language_status
391{
392 # The code of the target language (ensure it is lowercase)
393 my $target_language_code = lc(shift(@_));
394
395 # Check that the necessary arguments were supplied
396 if (!$target_language_code) {
397 &throw_fatal_error("Missing command argument.");
398 }
399
400 # Form an XML response to the command
401 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
402 $xml_response .= "<GTIResponse>\n";
403 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
404
405 foreach my $translation_file (@$gti_translation_files) {
406 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
407 my $target_file_name = "";
408
409 if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg
410 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
411 &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
412
413 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
414 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
415
416 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
417 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
418 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
419 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
420 }
421 else {
422 # Get (and check) the translation configuration
423 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
424 $target_file_name = $target_file;
425
426 # Parse the source language and target language files
427 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
428 my @source_file_lines = &read_file_lines($source_file_path);
429 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
430
431 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
432 my @target_file_lines = &read_file_lines($target_file_path);
433 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
434
435 # Filter out any automatically translated chunks
436 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
437 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
438 delete $source_file_key_to_line_mapping{$chunk_key};
439 delete $target_file_key_to_line_mapping{$chunk_key};
440 }
441 }
442
443 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
444 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
445
446 # Determine the target file chunks requiring translation
447 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
448
449 # Determine the target file chunks requiring updating
450 my @target_file_keys_requiring_updating = ();
451 if (-e $target_file_path) {
452 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
453 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
454 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
455 }
456
457 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
458 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
459 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
460 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
461 }
462
463 &log_message("Status of " . $translation_file->{'key'});
464 &log_message("Number of source chunks: " . $num_source_chunks);
465 &log_message("Number of target chunks: " . $num_target_chunks);
466 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
467 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
468
469 $xml_response .= " <TranslationFile"
470 . " key=\"" . $translation_file->{'key'} . "\""
471 . " target_file_path=\"" . $target_file_name . "\""
472 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
473 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
474 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
475 }
476
477 $xml_response .= " </LanguageStatus>\n";
478
479 $xml_response .= "</GTIResponse>\n";
480 return $xml_response;
481}
482
483
484sub search_chunks
485{
486 # The code of the target language (ensure it is lowercase)
487 my $target_language_code = lc(shift(@_));
488 # The key of the file to translate (ensure it is lowercase)
489 my $translation_file_key = lc(shift(@_));
490 # The query string
491 my $query_string = join(' ', @_);
492
493 # Check that the necessary arguments were supplied
494 if (!$target_language_code || !$translation_file_key || !$query_string) {
495 &throw_fatal_error("Missing command argument.");
496 }
497
498 my ($source_file, $target_file, $translation_file_type) = ();
499 my %source_file_key_to_text_mapping = ();
500 my %target_file_key_to_text_mapping = ();
501
502
503 if ($translation_file_key !~ m/^gs3/) {
504 # Get (and check) the translation configuration
505 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
506
507 # Parse the source language and target language files
508 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
509 my @source_file_lines = &read_file_lines($source_file_path);
510 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
511
512 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
513 my @target_file_lines = &read_file_lines($target_file_path);
514 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
515
516 # Filter out any automatically translated chunks
517 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
518 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
519 delete $source_file_key_to_line_mapping{$chunk_key};
520 delete $target_file_key_to_line_mapping{$chunk_key};
521 }
522 }
523
524 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
525 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
526 }
527 else {
528 # Not needed in this case
529 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
530 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
531 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
532 }
533
534 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
535 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
536
537 # Determine the target file chunks matching the query
538 my @target_file_keys_matching_query = ();
539 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
540 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
541 if ($target_file_text =~ /$query_string/i) {
542 # &log_message("Chunk with key $chunk_key matches query.");
543 push(@target_file_keys_matching_query, $chunk_key);
544 }
545 }
546
547 # Form an XML response to the command
548 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
549 $xml_response .= "<GTIResponse>\n";
550
551 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
552 foreach my $chunk_key (@target_file_keys_matching_query) {
553 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
554
555 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
556 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
557 $xml_response .= " </Chunk>\n";
558 }
559 $xml_response .= " </ChunksMatchingQuery>\n";
560
561 $xml_response .= "</GTIResponse>\n";
562 return $xml_response;
563}
564
565
566sub submit_translations
567{
568 # The code of the target language (ensure it is lowercase)
569 my $target_language_code = lc(shift(@_));
570 # The key of the file to translate (ensure it is lowercase)
571 my $translation_file_key = lc(shift(@_));
572 # The username of the translation submitter
573 my $submitter_username = shift(@_);
574 # Whether to submit a target chunk even if it hasn't changed
575 my $force_submission_flag = shift(@_);
576
577 # Check that the necessary arguments were supplied
578 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
579 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
580 die "\n";
581 }
582
583 my %source_file_key_to_text_mapping = ();
584 my %source_file_key_to_gti_comment_mapping = ();
585 my %target_file_key_to_text_mapping = ();
586 my %target_file_key_to_gti_comment_mapping = ();
587
588 my (@source_file_lines, @target_file_lines) = ();
589 my ($source_file, $target_file, $translation_file_type);
590
591
592 if ($translation_file_key !~ m/^gs3/) {
593 # Get (and check) the translation configuration
594 ($source_file, $target_file, $translation_file_type)
595 = &get_translation_configuration($target_language_code, $translation_file_key);
596
597 # Parse the source language and target language files
598 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
599 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
600 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
601 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
602
603 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
604 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
605 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
606 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
607 }
608 else {
609 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
610 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
611 }
612 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
613 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
614
615 # Submission date
616 my $day = (localtime)[3];
617 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
618 my $year = (localtime)[5] + 1900;
619 my $submission_date = "$day-$month-$year";
620
621 open(SUBMISSION, "-");
622 my @submission_lines = <SUBMISSION>;
623 close(SUBMISSION);
624
625 # Remove any nasty carriage returns
626 # &log_message("Submission:");
627 foreach my $submission_line (@submission_lines) {
628 $submission_line =~ s/\r$//;
629 #&log_message(" $submission_line");
630 }
631
632 my %source_file_key_to_submission_mapping = ();
633 my %target_file_key_to_submission_mapping = ();
634 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
635 # Read source file part of submission
636 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
637 my $chunk_key = $1;
638
639 # Read the source file text
640 my $source_file_chunk_text = "";
641 $i++;
642 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
643 $source_file_chunk_text .= $submission_lines[$i];
644 $i++;
645 }
646 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
647 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
648
649 #&log_message("Source file key: $chunk_key");
650 #&log_message("Source file text: $source_file_chunk_text");
651 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
652 }
653
654 # Read target file part of submission
655 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
656 my $chunk_key = $1;
657
658 # Read the target file text
659 my $target_file_chunk_text = "";
660 $i++;
661 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
662 $target_file_chunk_text .= $submission_lines[$i];
663 $i++;
664 }
665 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
666 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
667
668 #&log_message("Target file key: $chunk_key");
669 #&log_message("Target file text: $target_file_chunk_text");
670 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
671 }
672 }
673
674 # -----------------------------------------
675 # Validate the translation submissions
676 # -----------------------------------------
677
678 # Check that the translations are valid
679 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
680
681 # Kathy introduced escaped colons ("\:") into chunk keys in properties files (greenstone3/metadata_names),
682 # but they're not escaped in the submitted XML versions, nor are they escaped in memory (in the $chunk_key)
683
684 # Make sure the submitted chunk still exists in the source file
685 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
686 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
687 delete $source_file_key_to_submission_mapping{$chunk_key};
688 delete $target_file_key_to_submission_mapping{$chunk_key};
689 next;
690 }
691
692 # Make sure the submitted source chunk matches the source file chunk
693 if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
694 #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
695 #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
696 #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
697
698 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
699 &log_message("Submission source: |$source_file_key_to_submission_mapping{$chunk_key}|");
700 &log_message(" Source text: |$source_file_key_to_text_mapping{$chunk_key}|");
701 delete $source_file_key_to_submission_mapping{$chunk_key};
702 delete $target_file_key_to_submission_mapping{$chunk_key};
703 next;
704 }
705 }
706
707 # Apply the submitted translations
708 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
709 # Only apply the submission if it is a change, unless -force_submission has been specified
710 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
711 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
712 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
713 }
714 }
715
716 if ($translation_file_key !~ m/^gs3/) {
717 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
718 } else {
719 eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
720 }
721}
722
723
724sub create_glihelp_zip_file
725{
726 my $target_language_code = shift(@_);
727 my $translation_file_key = "glihelp";
728
729 &log_message("Creating GLI Help zip file for $target_language_code");
730
731 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
732
733 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
734 my $oldclasspath = $classpath;
735 if ( ! -e $classpath) {
736 $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
737 }
738 if ( ! -e $classpath) {
739 &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
740 }
741
742
743 my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
744 my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
745 my $os = $^O;
746 my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
747 my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
748 $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
749
750 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
751 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
752
753 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
754 if ( ! -e $gen_many_html_xsl_filepath) {
755 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
756 }
757
758 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
759 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
760
761 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
762 $target_file_directory = $target_file_directory."/";
763
764 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
765
766 # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
767 my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
768 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
769
770 # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
771 my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
772 my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
773 &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
774
775 # The following file reading section is a candidate to use FileUtils::readUTF8File()
776 # in place of calling sysread() directly. But only if we can reason we'd be working with UTF8
777 # In gli/help/<newlang>/help.xml, replace all occurrences of
778 # <Text id="1">This text in en will be removed for new langcode</Text>
779 # with <!-- Missing translation: 1 -->
780 open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
781 my $help_xml_contents;
782 # Read in the entire contents of the file in one hit
783 sysread(FIN, $help_xml_contents, -s FIN);
784 close(FIN);
785
786 $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
787
788 open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
789 print FOUT $help_xml_contents;
790 close(FOUT);
791 }
792
793 my $perl_exec = &util::get_perl_exec();
794 my $java_exec = "java";
795 if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
796 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
797 } elsif(defined($ENV{'JRE_HOME'}) && $ENV{'JRE_HOME'} ne ""){
798 $java_exec = &util::filename_cat($ENV{'JRE_HOME'}, "bin", "java");
799 }
800
801 #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
802 my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
803 #&throw_fatal_error("RAN gti command: $cmd");
804 my $response = `$cmd`;
805
806 #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
807 $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
808 $response = `$cmd`;
809
810 # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
811 my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
812 if(!&FileUtils::directoryExists($tmpdir)) {
813 &FileUtils::makeDirectory($tmpdir);
814 }
815 #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
816 my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
817 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
818
819 $response = `$cmd`;
820}
821
822
823sub get_translation_configuration
824{
825 # Get the code of the target language
826 my $target_language_code = shift(@_);
827 # Get the key of the file to translate
828 my $translation_file_key = shift(@_);
829
830 # Read the translation data from the gti.cfg file
831 my ($source_file, $target_file, $translation_file_type) =
832 &get_translation_data_for($target_language_code, $translation_file_key);
833
834 # Check that the file to translate is defined in the gti.cfg file
835 if (!$source_file || !$target_file || !$translation_file_type) {
836 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
837 }
838
839 # Check that the source file exists
840 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
841 if (!-e $source_file_path) {
842 &throw_fatal_error("Source file $source_file_path does not exist.");
843 }
844
845 # Check that the source file is up to date
846 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
847 # unless ($translation_file_is_not_in_cvs) {
848 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
849 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
850 if ($source_file_cvs_status =~ /^C /) {
851 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
852 }
853 if ($source_file_cvs_status =~ /^M /) {
854 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
855 }
856 # }
857
858 return ($source_file, $target_file, $translation_file_type);
859}
860
861
862sub get_translation_data_for
863{
864 my ($target_language_code, $translation_file_key) = @_;
865
866 foreach my $translation_file (@$gti_translation_files) {
867 # If this isn't the correct translation file, move onto the next one
868 next if ($translation_file_key ne $translation_file->{'key'});
869
870 # Resolve the target language file
871 my $target_language_file = $translation_file->{'target_file'};
872 if ($target_language_file =~ /(\{.+\;.+\})/) {
873 my $unresolved_target_language_file_part = $1;
874
875 # Check for a special case for the target language code
876 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
877 my $resolved_target_language_file_part = $2;
878 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
879 }
880 # Otherwise use the last part as the default value
881 else {
882 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
883 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
884 }
885 }
886
887 # Resolve instances of {iso_639_1_target_language_name}
888 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
889 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
890 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
891
892 # Resolve instances of {target_language_code}
893 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
894
895 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
896}
897
898return ();
899}
900
901
902sub read_file_lines
903{
904 my ($file_path) = @_;
905
906 if (!open(FILE_IN, "<$file_path")) {
907 &log_message("Note: Could not open file $file_path.");
908 return ();
909 }
910 my @file_lines = <FILE_IN>;
911 close(FILE_IN);
912
913 return @file_lines;
914}
915
916
917sub build_key_to_line_mapping
918{
919 my ($file_lines, $translation_file_type) = @_;
920 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
921}
922
923
924sub build_key_to_text_mapping
925{
926 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
927
928 my %key_to_text_mapping = ();
929 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
930 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
931 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
932
933 my $chunk_text = @$file_lines[$chunk_starting_line];
934 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
935 $chunk_text .= @$file_lines[$l];
936 }
937
938 # Map from chunk key to text
939 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
940
941 #if($chunk_key =~ m/document\\/) {
942 #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
943 #&log_message("@@@ chunk key: $chunk_key");
944 #}
945
946 }
947
948 return %key_to_text_mapping;
949}
950
951
952sub build_key_to_last_update_date_mapping
953{
954 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
955
956 # If the files aren't in CVS then we can't tell anything about what needs updating
957 # return () if ($translation_file_is_not_in_cvs);
958
959 # Build a mapping from key to CVS date
960 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
961 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
962
963 # Build a mapping from key to comment date
964 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
965
966 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
967 my %key_to_last_update_date_mapping = ();
968 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
969 # Use the CVS date as a starting point
970 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
971 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
972
973 # If a comment date exists and it is after the CVS date, use that instead
974 # need to convert the comment date format to SVN format
975 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
976 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
977 my $chunk_comment_date = $1;
978 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
979 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
980 }
981 }
982 }
983
984 return %key_to_last_update_date_mapping;
985}
986
987
988sub build_key_to_cvs_date_mapping
989{
990 my ($filename, $translation_file_type) = @_;
991
992 # Use SVN to annotate each line of the file with the date it was last edited
993 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
994 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
995
996 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
997
998 my @cvs_annotated_file_lines_date = ();
999 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
1000 # Extract the date from the SVN annotation at the front
1001 # svn format : 2007-07-16
1002 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
1003
1004 push(@cvs_annotated_file_lines_date, $1);
1005
1006 # trim extra date information in svn annotation format
1007 # 15:42:49 +1200 (Wed, 21 Jun 2006)
1008 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1009 }
1010
1011 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1012 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1013
1014 my %key_to_cvs_date_mapping = ();
1015 foreach my $chunk_key (keys(%key_to_line_mapping)) {
1016 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1017 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1018
1019 # Find the date this chunk was last edited, from the CVS annotation
1020 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
1021 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1022 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1023 # This part of the chunk has been updated more recently
1024 $chunk_date = $cvs_annotated_file_lines_date[$l];
1025
1026 }
1027 }
1028
1029 # Map from chunk key to CVS date
1030 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1031 }
1032
1033 return %key_to_cvs_date_mapping;
1034}
1035
1036
1037sub build_key_to_gti_comment_mapping
1038{
1039 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1040
1041 my %key_to_gti_comment_mapping = ();
1042 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1043 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1044 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1045
1046 my $chunk_text = @$file_lines[$chunk_starting_line];
1047 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1048 $chunk_text .= @$file_lines[$l];
1049 }
1050
1051 # Map from chunk key to GTI comment
1052 my $chunk_gti_comment;
1053 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1054 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1055 }
1056
1057 return %key_to_gti_comment_mapping;
1058}
1059
1060
1061sub determine_chunks_requiring_translation
1062{
1063 my $source_file_key_to_text_mapping = shift(@_);
1064 my $target_file_key_to_text_mapping = shift(@_);
1065
1066 # Chunks needing translation are those in the source file with no translation in the target file
1067 my @target_file_keys_requiring_translation = ();
1068 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1069 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1070 # &log_message("Chunk with key $chunk_key needs translating.");
1071 push(@target_file_keys_requiring_translation, $chunk_key);
1072 }
1073 }
1074
1075 return @target_file_keys_requiring_translation;
1076}
1077
1078
1079sub determine_chunks_requiring_updating
1080{
1081 my $source_file_key_to_last_update_date_mapping = shift(@_);
1082 my $target_file_key_to_last_update_date_mapping = shift(@_);
1083
1084 # Chunks needing updating are those in the target file that have been more recently edited in the source file
1085 my @target_file_keys_requiring_updating = ();
1086 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1087 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1088 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1089
1090 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1091
1092 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1093 # &log_message("Chunk with key $chunk_key needs updating.");
1094 # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1095 push(@target_file_keys_requiring_updating, $chunk_key);
1096 }
1097 }
1098
1099 return @target_file_keys_requiring_updating;
1100}
1101
1102
1103sub is_chunk_automatically_translated
1104{
1105 my ($chunk_key, $translation_file_type) = @_;
1106 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1107}
1108
1109
1110sub make_text_xml_safe
1111{
1112 my $text = shift(@_);
1113 $text =~ s/\&/\&amp\;/g;
1114 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1115 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1116 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1117 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1118 $text =~ s/</\&lt\;/g;
1119 $text =~ s/>/\&gt\;/g;
1120 return $text;
1121}
1122
1123
1124sub unmake_text_xml_safe
1125{
1126 my $text = shift(@_);
1127 $text =~ s/\&lt\;/</g;
1128 $text =~ s/\&gt\;/>/g;
1129 $text =~ s/\&amp\;/\&/g;
1130 return $text;
1131}
1132
1133
1134# Returns 1 if $date1 is after $date2, 0 otherwise
1135sub is_date_after_cvs
1136{
1137 my ($date1, $date2) = @_;
1138 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
1139 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1140
1141 if(!defined $date1) {
1142 return 1;
1143 }
1144
1145 my @date1parts = split(/-/, $date1);
1146 my @date2parts = split(/-/, $date2);
1147
1148 # Compare year - nasty because we have rolled over into a new century
1149 my $year1 = $date1parts[2];
1150 if ($year1 < 80) {
1151 $year1 += 2000;
1152 }
1153 my $year2 = $date2parts[2];
1154 if ($year2 < 80) {
1155 $year2 += 2000;
1156 }
1157
1158 # Compare year
1159 if ($year1 > $year2) {
1160 return 1;
1161 }
1162 elsif ($year1 == $year2) {
1163 # Year is the same, so compare month
1164 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1165 return 1;
1166 }
1167 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1168 # Month is the same, so compare day
1169 if ($date1parts[0] > $date2parts[0]) {
1170 return 1;
1171 }
1172 }
1173 }
1174
1175 return 0;
1176}
1177
1178sub is_date_after
1179{
1180 my ($date1, $date2) = @_;
1181
1182 if(!defined $date1) {
1183 return 1;
1184 }
1185 if(!defined $date2) {
1186 return 0;
1187 }
1188
1189 # 16-Aug-2006
1190 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1191 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1192 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1193 $date1=$3 . "-" . $months{$2} . "-" . $1;
1194 # print "** converted date1: $date1\n";
1195 }
1196 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1197 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1198 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1199 $date2=$3 . "-" . $months{$2} . "-" . $1;
1200 # print "** converted date2: $date2\n";
1201 }
1202
1203
1204 # 2006-08-16
1205 my @date1parts = split(/-/, $date1);
1206 my @date2parts = split(/-/, $date2);
1207
1208 # Compare year
1209 if ($date1parts[0] > $date2parts[0]) {
1210 return 1;
1211 }
1212 elsif ($date1parts[0] == $date2parts[0]) {
1213 # Year is the same, so compare month
1214 if ($date1parts[1] > $date2parts[1]) {
1215 return 1;
1216 }
1217 elsif ($date1parts[1] == $date2parts[1]) {
1218 # Month is the same, so compare day
1219 if ($date1parts[2] > $date2parts[2]) {
1220 return 1;
1221 }
1222 }
1223 }
1224
1225 return 0;
1226}
1227
1228
1229sub create_xml_response_for_chunks_requiring_work
1230{
1231 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1232
1233 # Form an XML response to the command
1234 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1235 $xml_response .= "<GTIResponse>\n";
1236 $xml_response .= " <TranslationFile"
1237 . " key=\"" . $translation_file_key . "\""
1238 . " target_file_path=\"" . $target_file . "\""
1239 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1240 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1241 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1242
1243 # Do chunks requiring translation first
1244 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1245 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1246 }
1247 else {
1248 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1249 }
1250
1251 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1252 foreach my $chunk_key (@sorted_chunk_keys) {
1253 last if ($num_chunks_to_return == 0);
1254
1255 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1256 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1257
1258 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1259 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1260 $xml_response .= " <TargetFileText></TargetFileText>\n";
1261 $xml_response .= " </Chunk>\n";
1262
1263 $num_chunks_to_return--;
1264 }
1265
1266 $xml_response .= " </ChunksRequiringTranslation>\n";
1267
1268 # Then do chunks requiring updating
1269 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1270 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1271 }
1272 else {
1273 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1274 }
1275
1276 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1277 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1278 foreach my $chunk_key (@sorted_chunk_keys) {
1279 last if ($num_chunks_to_return == 0);
1280
1281 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1282 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1283 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1284 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1285
1286 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1287 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1288 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1289 $xml_response .= " </Chunk>\n";
1290
1291 $num_chunks_to_return--;
1292 }
1293
1294 $xml_response .= " </ChunksRequiringUpdating>\n";
1295
1296 $xml_response .= "</GTIResponse>\n";
1297
1298 return $xml_response;
1299}
1300
1301sub create_xml_response_for_uptodate_chunks
1302{
1303 my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1304
1305 # Form an XML response to the command
1306 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1307 $xml_response .= "<GTIResponse>\n";
1308 $xml_response .= " <TranslationFile"
1309 . " key=\"" . $translation_file_key . "\""
1310 . " target_file_path=\"" . $target_file . "\""
1311 . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1312
1313
1314 # Then do chunks requiring updating
1315 $xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1316
1317
1318 # foreach my $chunk_key (@uptodate_target_file_keys) {
1319 my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1320 foreach my $chunk_key (@sorted_chunk_keys) {
1321
1322 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1323 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1324 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1325 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1326
1327 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1328 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1329 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1330 $xml_response .= " </Chunk>\n";
1331
1332 }
1333
1334 $xml_response .= " </UptodateChunks>\n";
1335
1336 $xml_response .= "</GTIResponse>\n";
1337
1338 return $xml_response;
1339}
1340
1341sub create_xml_response_for_all_chunks
1342{
1343 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1344
1345 # Form an XML response to the command
1346 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1347 $xml_response .= "<GTIResponse>\n";
1348 $xml_response .= " <TranslationFile"
1349 . " key=\"" . $translation_file_key . "\""
1350 . " target_file_path=\"" . $target_file . "\"\/>\n";
1351
1352 # Do all the chunks
1353 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1354
1355 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1356 foreach my $chunk_key (@sorted_chunk_keys) {
1357 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1358 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1359
1360 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1361 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1362 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1363 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1364 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1365 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1366 }
1367 else {
1368 $xml_response .= " <TargetFileText></TargetFileText>\n";
1369 }
1370
1371 $xml_response .= " </Chunk>\n";
1372 }
1373 $xml_response .= " </Chunks>\n";
1374
1375 $xml_response .= "</GTIResponse>\n";
1376 return $xml_response;
1377}
1378
1379
1380
1381# ==========================================================================================
1382# MACROFILE FUNCTIONS
1383
1384sub build_key_to_line_mapping_for_macrofile
1385{
1386 my (@file_lines) = @_;
1387
1388 my $macro_package;
1389 my %chunk_key_to_line_mapping = ();
1390 # Process the contents of the file, line by line
1391 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1392 my $line = $file_lines[$i];
1393 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1394
1395 # Check if a new package is being defined
1396 if ($line =~ m/^package\s+(.+)/) {
1397 $macro_package = $1;
1398 }
1399
1400 # Line contains a macro name
1401 elsif ($line =~ m/^(_\w+_)/) {
1402 my $macro_key = $1;
1403 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1404
1405 # While there is still text of the macro to go...
1406 my $startline = $i;
1407 while ($line !~ /\}$/) {
1408 $i++;
1409 if ($i == scalar(@file_lines)) {
1410 &throw_fatal_error("Could not find end of macro $macro_key.");
1411 }
1412 $line = $file_lines[$i];
1413 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1414 }
1415
1416 # The chunk key consists of the package name and the macro key
1417 my $chunk_key = $macro_package . "." . $macro_key;
1418 # Map from chunk key to line
1419 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1420 }
1421
1422 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1423 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1424 # The chunk key consists of package name and macro key
1425 my $chunk_key = $macro_package . "." . $1;
1426 # Map from chunk key to line
1427 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1428}
1429}
1430
1431return %chunk_key_to_line_mapping;
1432}
1433
1434
1435sub import_chunk_from_macrofile
1436{
1437 my ($chunk_text) = @_;
1438
1439 # Is this an icon macro??
1440 if ($chunk_text =~ /^\#\# (.*)/) {
1441 # Extract image macro text
1442 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1443 $chunk_text = $1;
1444
1445 # Remove enclosing quotes
1446 $chunk_text =~ s/^\"//;
1447 $chunk_text =~ s/\"$//;
1448 }
1449
1450 # No, so it must be a text macro
1451 else {
1452 # Remove macro key
1453 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1454
1455 # Remove language specifier
1456 $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1457
1458 # Remove braces enclosing text
1459 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1460 }
1461
1462 return $chunk_text;
1463}
1464
1465
1466sub get_macrofile_chunk_gti_comment
1467{
1468 my ($chunk_text) = @_;
1469
1470 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1471 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1472 return $1;
1473}
1474
1475return undef;
1476}
1477
1478
1479sub is_macrofile_chunk_automatically_translated
1480{
1481 my ($chunk_key) = @_;
1482
1483 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1484 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1485 return 1;
1486 }
1487
1488 return 0;
1489}
1490
1491
1492# Use the source file to generate a target file that is formatted the same
1493sub write_translated_macrofile
1494{
1495 my $source_file = shift(@_); # Not used
1496 my @source_file_lines = @{shift(@_)};
1497 my $source_file_key_to_text_mapping = shift(@_);
1498 my $target_file = shift(@_);
1499 my @target_file_lines = @{shift(@_)};
1500 my $target_file_key_to_text_mapping = shift(@_);
1501 my $target_file_key_to_gti_comment_mapping = shift(@_);
1502 my $target_language_code = shift(@_);
1503
1504 # Build a mapping from source file line to chunk key
1505 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1506 my %source_file_line_to_key_mapping = ();
1507 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1508 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1509 }
1510 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1511 my $source_file_line_number = 0;
1512
1513 # Build a mapping from target file line to chunk key
1514 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1515 my %target_file_line_to_key_mapping = ();
1516 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1517 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1518 }
1519 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1520
1521 # Write the new target file
1522 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1523 if (!open(TARGET_FILE, ">$target_file_path")) {
1524 &throw_fatal_error("Could not write target file $target_file_path.");
1525 }
1526
1527 # Use the header from the target file, to keep language and author information
1528 if (scalar(@target_file_line_keys) > 0) {
1529 my $target_file_line_number = 0;
1530 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1531 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1532 my $target_file_line = $target_file_lines[$target_file_line_number];
1533 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1534 print TARGET_FILE $target_file_line;
1535 $target_file_line_number++;
1536 }
1537
1538 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1539 }
1540
1541 # Model the new target file on the source file, with the target file translations
1542 foreach my $line_key (@source_file_line_keys) {
1543 # Fill in the gaps before this chunk starts
1544 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1545 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1546 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1547 print TARGET_FILE $source_file_lines[$source_file_line_number];
1548 $source_file_line_number++;
1549 }
1550 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1551
1552 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1553 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1554 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1555
1556 my $macrofile_key = $chunk_key;
1557 $macrofile_key =~ s/^(.+?)\.//;
1558
1559 # If no translation exists for this chunk, show this, and move on
1560 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1561 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1562 next;
1563 }
1564
1565 # Grab the source chunk text
1566 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1567 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1568 $source_file_chunk .= $source_file_lines[$l];
1569 }
1570
1571 # Is this an icon macro??
1572 if ($source_file_chunk =~ /^\#\# (.*)/) {
1573 # Escape any newline and question mark characters so the source text is replaced correctly
1574 $source_file_chunk_text =~ s/\\/\\\\/g;
1575 $source_file_chunk_text =~ s/\?/\\\?/g;
1576
1577 # Build the new target chunk from the source chunk
1578 my $target_file_chunk = $source_file_chunk;
1579 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1580 $target_file_chunk =~ s/(\s)*$//;
1581 print TARGET_FILE "$target_file_chunk";
1582 }
1583
1584 # No, it is just a normal text macro
1585 else {
1586 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1587 }
1588
1589 # Add the "updated" comment, if one exists
1590 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1591 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1592 }
1593 print TARGET_FILE "\n";
1594}
1595
1596close(TARGET_FILE);
1597}
1598
1599
1600sub sort_by_line
1601{
1602 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1603}
1604
1605
1606# ==========================================================================================
1607# RESOURCE BUNDLE FUNCTIONS
1608
1609# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1610sub build_key_to_line_mapping_for_resource_bundle
1611{
1612 my (@file_lines) = @_;
1613
1614 my %chunk_key_to_line_mapping = ();
1615
1616 my $chunk_key;
1617 my $startindex = -1;
1618
1619 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1620 my $line = $file_lines[$i];
1621 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1622
1623 # a property line has a colon/equals sign as separator that is NOT escaped with a backslash (both keys and values
1624 # can use the colon or = sign. But in the key, such a char is always escaped. Unfortunately, they've not always been
1625 # escaped in the values. So we get the left most occurrence by not doing a greedy match (use ? to not be greedy).
1626 # So find the first :/= char not preceded by \. That will be the true separator of a chunk_key and its value chunk_text
1627
1628 if ($line =~ m/^(\S*?[^\\])[:|=](.*)$/) {
1629 # Line contains a dictionary string
1630
1631 # Unused but useful: http://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
1632 # http://perldoc.perl.org/perlvar.html
1633
1634 $chunk_key = $1;
1635 # remove the escaping of any :/= property separator from the chunk_key in memory,
1636 # to make comparison with its unescaped version during submissions easier. Will write out with escaping.
1637 $chunk_key =~ s/\\([:=])/$1/g;
1638
1639 $startindex = $i;
1640 }
1641 if ($startindex != -1) {
1642 if($line !~ m/\\$/) { # line finished
1643 # $i keeps track of the line at which this property (chunk_key) finishes
1644
1645 # Map from chunk key to line
1646 $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1647 $startindex = -1;
1648 $chunk_key = "";
1649 }
1650 }
1651 }
1652
1653 return %chunk_key_to_line_mapping;
1654}
1655
1656
1657sub import_chunk_from_resource_bundle
1658{
1659 my ($chunk_text) = @_;
1660
1661 # Simple: just remove string key.
1662 # But key can contain an escaped separator (\: or \=).
1663 # So just as in the previous subroutine, find the first (leftmost) : or = char not preceded by \.
1664 # That will be the true separator of a chunk_key and its value chunk_text
1665 $chunk_text =~ s/^(\S*?[^\\])[:|=](\s*)//s;
1666
1667 $chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
1668 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1669
1670 return $chunk_text;
1671}
1672
1673
1674sub get_resource_bundle_chunk_gti_comment
1675{
1676 my ($chunk_text) = @_;
1677
1678 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1679 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1680 return $1;
1681 }
1682
1683 return undef;
1684}
1685
1686
1687sub is_resource_bundle_chunk_automatically_translated
1688{
1689 # No resource bundle chunks are automatically translated
1690 return 0;
1691}
1692
1693
1694sub write_translated_resource_bundle
1695{
1696 my $source_file = shift(@_); # Not used
1697 my @source_file_lines = @{shift(@_)};
1698 my $source_file_key_to_text_mapping = shift(@_);
1699 my $target_file = shift(@_);
1700 my @target_file_lines = @{shift(@_)}; # Not used
1701 my $target_file_key_to_text_mapping = shift(@_);
1702 my $target_file_key_to_gti_comment_mapping = shift(@_);
1703 my $target_language_code = shift(@_); # Not used
1704
1705 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1706 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1707 my %source_file_line_to_key_mapping = ();
1708 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1709 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1710 }
1711
1712 # Write the new target file
1713 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1714 if (!open(TARGET_FILE, ">$target_file_path")) {
1715 &throw_fatal_error("Could not write target file $target_file_path.");
1716 }
1717
1718 # Model the new target file on the source file, with the target file translations
1719 my $source_file_line_number = 0;
1720 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1721 # Fill in the gaps before this chunk starts
1722 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1723 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1724 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1725 print TARGET_FILE $source_file_lines[$source_file_line_number];
1726 $source_file_line_number++;
1727 }
1728 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1729
1730 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1731 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1732 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1733
1734 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
1735 # since the key-value separator in a property resource bundle file is : or =
1736 my $escaped_chunk_key = $chunk_key;
1737 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
1738
1739 # If no translation exists for this chunk, show this, and move on
1740 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1741 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
1742 next;
1743 }
1744
1745 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
1746 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1747 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1748 }
1749 print TARGET_FILE "\n";
1750 }
1751
1752 close(TARGET_FILE);
1753}
1754
1755
1756# ==========================================================================================
1757# GREENSTONE XML FUNCTIONS
1758
1759sub build_key_to_line_mapping_for_greenstone_xml
1760{
1761 my (@file_lines) = @_;
1762
1763 my %chunk_key_to_line_mapping = ();
1764 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1765 my $line = $file_lines[$i];
1766 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1767
1768 # Line contains a string to translate
1769 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1770 my $chunk_key = $1;
1771 $line =~ s/\s*$//; # Remove any nasty whitespace
1772 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1773
1774 # While there is still text of the string to go...
1775 my $startline = $i;
1776 while ($line !~ /<\/Text>$/) {
1777 $i++;
1778 if ($i == scalar(@file_lines)) {
1779 &throw_fatal_error("Could not find end of string $chunk_key.");
1780 }
1781 $line = $file_lines[$i];
1782 $line =~ s/\s*$//; # Remove any nasty whitespace
1783 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1784 }
1785
1786 # Map from chunk key to line
1787 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1788 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1789 }
1790 else {
1791 &throw_fatal_error("Duplicate key $chunk_key.");
1792 }
1793 }
1794 }
1795
1796 return %chunk_key_to_line_mapping;
1797}
1798
1799
1800sub import_chunk_from_greenstone_xml
1801{
1802 my ($chunk_text) = @_;
1803
1804 # Simple: just remove the Text tags
1805 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1806 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1807 $chunk_text =~ s/<\/Text>$//;
1808
1809 return $chunk_text;
1810}
1811
1812
1813sub get_greenstone_xml_chunk_gti_comment
1814{
1815 my ($chunk_text) = @_;
1816
1817 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1818 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1819 return $1;
1820 }
1821
1822 return undef;
1823}
1824
1825
1826sub is_greenstone_xml_chunk_automatically_translated
1827{
1828 # No greenstone XML chunks are automatically translated
1829 return 0;
1830}
1831
1832
1833sub write_translated_greenstone_xml
1834{
1835 my $source_file = shift(@_); # Not used
1836 my @source_file_lines = @{shift(@_)};
1837 my $source_file_key_to_text_mapping = shift(@_);
1838 my $target_file = shift(@_);
1839 my @target_file_lines = @{shift(@_)}; # Not used
1840 my $target_file_key_to_text_mapping = shift(@_);
1841 my $target_file_key_to_gti_comment_mapping = shift(@_);
1842 my $target_language_code = shift(@_); # Not used
1843
1844 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1845 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1846 my %source_file_line_to_key_mapping = ();
1847 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1848 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1849 }
1850
1851 # Write the new target file
1852 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1853 if (!open(TARGET_FILE, ">$target_file_path")) {
1854 &throw_fatal_error("Could not write target file $target_file_path.");
1855 }
1856
1857 # Model the new target file on the source file, with the target file translations
1858 my $source_file_line_number = 0;
1859 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1860 # Fill in the gaps before this chunk starts
1861 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1862 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1863 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1864 print TARGET_FILE $source_file_lines[$source_file_line_number];
1865 $source_file_line_number++;
1866 }
1867 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1868
1869 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1870 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1871 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1872 $target_file_chunk_text =~ s/(\n)*$//g;
1873
1874 # If no translation exists for this chunk, show this, and move on
1875 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1876 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1877 next;
1878 }
1879
1880 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1881 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1882 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1883 $chunk_gti_comment =~ s/^Updated //;
1884 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1885 }
1886 print TARGET_FILE "\n";
1887 }
1888
1889 # Fill in the end of the file
1890 while ($source_file_line_number < scalar(@source_file_lines)) {
1891 print TARGET_FILE $source_file_lines[$source_file_line_number];
1892 $source_file_line_number++;
1893 }
1894
1895 close(TARGET_FILE);
1896}
1897
1898
1899# ==========================================================================================
1900# GREENSTONE3 FUNCTIONS
1901
1902sub get_all_chunks_for_gs3
1903{
1904 # The code of the target language (ensure it is lowercase)
1905 my $target_language_code = lc(shift(@_));
1906 my $translation_file_key = lc(shift(@_));
1907
1908 # Check that the necessary arguments were supplied
1909 if (!$target_language_code) {
1910 &throw_fatal_error("Missing command argument.");
1911 }
1912
1913 # Get (and check) the translation configuration
1914 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1915
1916 my %source_files_key_to_text_mapping = ();
1917 my %target_files_key_to_text_mapping = ();
1918 my %source_files_key_to_last_update_date_mapping = ();
1919 my %target_files_key_to_last_update_date_mapping = ();
1920
1921 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1922
1923 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1924 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1925
1926 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1927 return $xml_response;
1928}
1929
1930
1931sub get_first_n_chunks_requiring_work_for_gs3
1932{
1933 # The code of the target language (ensure it is lowercase)
1934 my $target_language_code = lc(shift(@_));
1935 # The key of the file to translate (ensure it is lowercase)
1936 my $translation_file_key = lc(shift(@_));
1937 # The number of chunks to return (defaults to one if not specified)
1938 my $num_chunks_to_return = shift(@_) || "1";
1939
1940 # Check that the necessary arguments were supplied
1941 if (!$target_language_code || !$translation_file_key) {
1942 &throw_fatal_error("Missing command argument.");
1943 }
1944
1945 my %source_files_key_to_text_mapping = ();
1946 my %target_files_key_to_text_mapping = ();
1947 my %source_files_key_to_last_update_date_mapping = ();
1948 my %target_files_key_to_last_update_date_mapping = ();
1949
1950 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1951 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1952
1953 # Determine the target file chunks requiring translation
1954 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1955 # Determine the target file chunks requiring updating
1956 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1957 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1958 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1959
1960 my $download_target_filepath = "";
1961
1962
1963 # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1964
1965 # if there is no copy of the language files for download, there's also no link to the spreadsheet
1966 # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1967 # interface *.properties files,and link to that zip as the file for offline translation.
1968 # Selecting only properties files for English and the language they're working on (if the last exists)
1969
1970 # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1971 # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1972
1973 my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1974 my $tar_cmd = "tar -cvzf $zip";
1975
1976
1977 # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1978 my $curdir = `pwd`;
1979 chdir $gsdl_root_directory;
1980
1981 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
1982 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
1983
1984 # tar command will overwrite the previous version, but want to check we've created it
1985 if(&FileUtils::fileExists($zip)) {
1986 &FileUtils::removeFiles($zip);
1987 }
1988
1989 #my $tar_result = system($tar_cmd); # works but then interface breaks
1990 `$tar_cmd`;
1991 my $tar_result = $?;
1992
1993 if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1994 $download_target_filepath = $zip;
1995 } else {
1996 &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1997 }
1998
1999 # change back to original working directory (cgi-bin/linux probably)
2000 chdir $curdir;
2001
2002 # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
2003
2004
2005 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
2006 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
2007 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2008 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2009
2010 return $xml_response;
2011}
2012
2013# helper function
2014# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg)
2015# formatted correctly to go into a zip file
2016sub get_gs3_zip_file_listing
2017{
2018 my $target_language_code = shift(@_);
2019 my $sourcedir = shift(@_);
2020 my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
2021
2022 my $filelisting = "";
2023 foreach my $interface_file (@$files_array) {
2024
2025 my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
2026 my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2027
2028 $filelisting = "$filelisting $source_filepath";
2029 if(&FileUtils::fileExists($target_filepath)) {
2030 $filelisting = "$filelisting $target_filepath";
2031 }
2032 }
2033
2034 return $filelisting;
2035}
2036
2037sub get_uptodate_chunks_for_gs3
2038{
2039 # The code of the target language (ensure it is lowercase)
2040 my $target_language_code = lc(shift(@_));
2041 # The key of the file to translate (ensure it is lowercase)
2042 my $translation_file_key = lc(shift(@_));
2043 # The number of chunks to return (defaults to one if not specified)
2044 my $num_chunks_to_return = shift(@_) || "1";
2045
2046 # Check that the necessary arguments were supplied
2047 if (!$target_language_code || !$translation_file_key) {
2048 &throw_fatal_error("Missing command argument.");
2049 }
2050
2051 my %source_files_key_to_text_mapping = ();
2052 my %target_files_key_to_text_mapping = ();
2053 my %source_files_key_to_last_update_date_mapping = ();
2054 my %target_files_key_to_last_update_date_mapping = ();
2055
2056 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2057 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2058
2059
2060 # Chunks needing updating are those in the target file that have been more recently edited in the source file
2061 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2062 my @uptodate_target_file_keys = ();
2063 foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2064 my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2065 my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2066
2067 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
2068
2069 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2070 # &log_message("Chunk with key $chunk_key needs updating.");
2071 push(@uptodate_target_file_keys, $chunk_key);
2072 }
2073 }
2074
2075 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2076
2077 return $xml_response;
2078}
2079
2080
2081sub build_gs3_configuration
2082{
2083 my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2084 $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2085
2086 my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2087 my $files_array = \@gs3_interface_files;
2088
2089 if($translation_file_key eq "gs3colcfg") {
2090 $source_file_directory = "gs3-collection-configs";
2091 $files_array = \@gs3_col_cfg_files;
2092 }
2093 my $translation_file_type = "resource_bundle";
2094
2095 foreach my $interface_file_key (@$files_array) {
2096
2097 &log_message("Greenstone 3 interface file: " . $interface_file_key);
2098
2099 # Parse the source language and target language files
2100 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2101 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2102 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2103 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2104 #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2105
2106 my %source_file_key_to_gti_comment_or_last_updated_mapping;
2107 if($get_gti_comments_not_last_updated) {
2108 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2109 } else {
2110 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2111 }
2112
2113 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2114 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2115 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2116 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2117 #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2118
2119 my %target_file_key_to_gti_comment_or_last_updated_mapping;
2120 if($get_gti_comments_not_last_updated) {
2121 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2122 } else {
2123 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2124 }
2125
2126
2127 # Filter out any automatically translated chunks
2128 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2129 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2130 delete $source_file_key_to_line_mapping{$chunk_key};
2131 delete $target_file_key_to_line_mapping{$chunk_key};
2132 }
2133 }
2134
2135 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2136 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2137
2138 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2139 my $global_chunk_key = "$interface_file_key.$chunk_key";
2140 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2141 $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2142
2143 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2144 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2145 $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2146 }
2147 }
2148 }
2149}
2150
2151
2152sub write_translated_gs3interface
2153{
2154 my $translation_file_key = shift(@_);
2155 my $source_file_key_to_text_mapping = shift(@_);
2156 my $target_file_key_to_text_mapping = shift(@_);
2157 my $target_file_key_to_gti_comment_mapping = shift(@_);
2158 my $target_language_code = shift(@_);
2159
2160 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2161
2162 my %translated_interface_file_keys = ();
2163 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2164 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2165 if (!defined $translated_interface_file_keys{$1}) {
2166 &log_message("Updated interface file: " . $1);
2167 $translated_interface_file_keys{$1}="";
2168 }
2169 }
2170 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2171
2172 my $source_file_directory = "greenstone3";
2173 $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2174
2175 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2176
2177 # Build a mapping from chunk key to source file line, and from source file line to chunk key
2178 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2179 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2180 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2181 my %source_file_line_to_key_mapping = ();
2182 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2183 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2184 }
2185
2186 # Write the new target file
2187 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2188 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2189 if (!open(TARGET_FILE, ">$target_file_path")) {
2190 &throw_fatal_error("Could not write target file $target_file_path.");
2191 }
2192
2193 # Model the new target file on the source file, with the target file translations
2194 my $source_file_line_number = 0;
2195 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2196 # Fill in the gaps before this chunk starts
2197 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2198 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2199 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2200 print TARGET_FILE $source_file_lines[$source_file_line_number];
2201 $source_file_line_number++;
2202 }
2203 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2204
2205 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2206 my $global_chunk_key = "$interface_file_key.$chunk_key";
2207 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2208 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2209
2210 # make sure any : or = sign in the chunk key is escaped again (with \) when written out
2211 # since the key-value separator in a property resource bundle file is : or =
2212 my $escaped_chunk_key = $chunk_key;
2213 $escaped_chunk_key =~ s/(:|=)/\\$1/g; #$escaped_chunk_key =~ s/([^\\])(:|=)/\\$1$2/g;
2214
2215 # If no translation exists for this chunk, show this, and move on
2216 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2217 print TARGET_FILE "# -- Missing translation: $escaped_chunk_key\n";
2218 next;
2219 }
2220
2221 print TARGET_FILE "$escaped_chunk_key:$target_file_chunk_text";
2222 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2223 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2224 }
2225 print TARGET_FILE "\n";
2226 }
2227
2228 close(TARGET_FILE);
2229 }
2230}
2231
2232&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.