source: main/trunk/greenstone2/bin/script/gti.pl@ 30681

Last change on this file since 30681 was 30681, checked in by ak19, 8 years ago

3 new strings introduced by Kathy contained the :, which is used as a separator in the properties file. Although Kathy tried to escape it with a backslash, it broke GTI because GTI doesn't recognise the backslash as a separator and all kinds of weird things happened from then on, so that the Gujarati translator kept having to translte the current date rather than a real GS3 interface string. Modified the gti.pl code (to be committed) and the new strings that Kathy introduced, so that hopefully, GTI can now handle it. Property names and values will be split at the right-most separator character now (= or :) and any on the left should not be escaped.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 93.8 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gti.pl
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2005 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30BEGIN {
31 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
32 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
33}
34
35
36use iso639;
37use strict;
38use util;
39use FileUtils;
40
41my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
42my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
43my $source_language_code = "en"; # This is non-negotiable
44
45my $gti_translation_files =
46[ # Greenstone macrofiles
47{ 'key' => "coredm",
48 'file_type' => "macrofile",
49 'source_file' => "macros/english.dm",
50 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
51
52{ 'key' => "auxdm",
53 'file_type' => "macrofile",
54 'source_file' => "macros/english2.dm",
55 'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
56
57#{ 'key' => "paperspastdm",
58# 'file_type' => "macrofile",
59# 'source_file' => "macros/paperspast-english.dm",
60# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
61
62# GLI dictionary
63{ 'key' => "glidict",
64 'file_type' => "resource_bundle",
65 'source_file' => "gli/classes/dictionary.properties",
66 'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
67
68# GLI help
69{ 'key' => "glihelp",
70 'file_type' => "greenstone_xml",
71 'source_file' => "gli/help/en/help.xml",
72 'target_file' => "gli/help/{target_language_code}/help.xml" },
73
74# Greenstone Perl modules
75{ 'key' => "perlmodules",
76 'file_type' => "resource_bundle",
77 'source_file' => "perllib/strings.properties",
78 'target_file' => "perllib/strings_{target_language_code}.properties" },
79
80# Greenstone Installer interface
81{ 'key' => "gsinstaller",
82 'file_type' => "resource_bundle",
83 'source_file' => "gsinstaller/LanguagePack.properties",
84 'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
85
86# Greenstone tutorial exercises
87# { 'key' => "tutorials",
88# 'file_type' => "greenstone_xml",
89# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
90# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
91
92# new Greenstone.org
93{ 'key' => "greenorg",
94 'file_type' => "resource_bundle",
95 'source_file' => "greenstoneorg/website/classes/Gsc.properties",
96 'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
97},
98
99# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
100# check it out as greenstone3
101{ 'key' => "gs3interface",
102 'file_type' => "resource_bundle",
103 'source_file' => "greenstone3",
104 'target_file' => "greenstone3"
105},
106
107# collection config display items of GS3 demo collections. Checked out as gs3-collection-configs
108# from http://svn.greenstone.org/main/trunk/gs3-collection-configs
109{ 'key' => "gs3colcfg",
110 'file_type' => "resource_bundle",
111 'source_file' => "gs3-collection-configs",
112 'target_file' => "gs3-collection-configs"
113}
114];
115
116my @gs3_col_cfg_files = ("lucene-jdbm-demo", "solr-jdbm-demo");
117
118my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
119#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
120
121# Auxilliary GS3 interface files. This list is not used at present
122# Combine with above list if generating translation spreadsheet for all interface files
123my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
124
125my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
126
127# Not: i18n, log4j
128
129sub main
130{
131 # Get the command to process, and any arguments
132 my $gti_command = shift(@_);
133 my @gti_command_arguments = @_;
134 my $module = $_[1];
135
136 # for GS3, set gsdl_root_dir to GSDL3HOME
137 #if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
138 #if($ENV{'GSDL3SRCHOME'}) {
139 # $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
140 # $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
141 #}
142 #}
143
144 # Open the GTI log file for appending, or write to STDERR if that fails
145 if (!open(GTI_LOG, ">>$gti_log_file")) {
146 open(GTI_LOG, ">&STDERR");
147 }
148
149 # Log the command that launched this script
150 &log_message("Command: $0 @ARGV");
151
152 # Check that a command was supplied
153 if (!$gti_command) {
154 &throw_fatal_error("Missing command.");
155 }
156
157 # Process the command
158 if ($gti_command =~ /^get-all-chunks$/i) {
159 # Check that GS3 interface is the target
160 if ($module =~ m/^gs3/) { # gs3interface, gs3colcfg
161 print &get_all_chunks_for_gs3(@gti_command_arguments);
162 } else {
163 print &get_all_chunks(@gti_command_arguments);
164 }
165 }
166 elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
167 if ($module =~ m/^gs3/) {
168 print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
169 } else {
170 print &get_first_n_chunks_requiring_work(@gti_command_arguments);
171 }
172 }
173 elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
174 if ($module =~ m/^gs3/) {
175 print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
176 } else {
177 print &get_uptodate_chunks(@gti_command_arguments);
178 }
179 }
180 elsif ($gti_command =~ /^get-language-status$/i) {
181 print &get_language_status(@gti_command_arguments);
182 }
183 elsif ($gti_command =~ /^search-chunks$/i) {
184 print &search_chunks(@gti_command_arguments);
185 }
186 elsif ($gti_command =~ /^submit-translations$/i) {
187 # This command cannot produce any output since it reads input
188 &submit_translations(@gti_command_arguments);
189 }
190 elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
191 # This command cannot produce any output since it reads input
192 &create_glihelp_zip_file(@gti_command_arguments);
193 }
194 else {
195 # The command was not recognized
196 &throw_fatal_error("Unknown command \"$gti_command\".");
197 }
198}
199
200
201sub throw_fatal_error
202{
203 my $error_message = shift(@_);
204
205 # Write an XML error response
206 print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
207 print "<GTIResponse>\n";
208 print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
209 print "</GTIResponse>\n";
210
211 # Log the error message, then die
212 &log_message("Error: $error_message");
213 die "\n";
214}
215
216
217sub log_message
218{
219 my $log_message = shift(@_);
220 print GTI_LOG time() . " -- " . $log_message . "\n";
221}
222
223
224sub get_all_chunks
225{
226 # The code of the target language (ensure it is lowercase)
227 my $target_language_code = lc(shift(@_));
228 # The key of the file to translate (ensure it is lowercase)
229 my $translation_file_key = lc(shift(@_));
230
231 # Check that the necessary arguments were supplied
232 if (!$target_language_code || !$translation_file_key) {
233 &throw_fatal_error("Missing command argument.");
234 }
235
236 # Get (and check) the translation configuration
237 my ($source_file, $target_file, $translation_file_type)
238 = &get_translation_configuration($target_language_code, $translation_file_key);
239
240 # Parse the source language and target language files
241 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
242 my @source_file_lines = &read_file_lines($source_file_path);
243 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
244
245 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
246 my @target_file_lines = &read_file_lines($target_file_path);
247 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
248
249 # Filter out any automatically translated chunks
250 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
251 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
252 delete $source_file_key_to_line_mapping{$chunk_key};
253 delete $target_file_key_to_line_mapping{$chunk_key};
254 }
255 }
256
257 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
258 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
259 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
260 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
261
262 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
263 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
264
265 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
266
267 return $xml_response;
268}
269
270
271sub get_uptodate_chunks
272{
273 # The code of the target language (ensure it is lowercase)
274 my $target_language_code = lc(shift(@_));
275 # The key of the file to translate (ensure it is lowercase)
276 my $translation_file_key = lc(shift(@_));
277
278 # Check that the necessary arguments were supplied
279 if (!$target_language_code || !$translation_file_key) {
280 &throw_fatal_error("Missing command argument.");
281 }
282
283 # Get (and check) the translation configuration
284 my ($source_file, $target_file, $translation_file_type)
285 = &get_translation_configuration($target_language_code, $translation_file_key);
286
287 # Parse the source language and target language files
288 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
289 my @source_file_lines = &read_file_lines($source_file_path);
290 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
291
292 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
293 my @target_file_lines = &read_file_lines($target_file_path);
294 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
295
296 # Filter out any automatically translated chunks
297 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
298 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
299 delete $source_file_key_to_line_mapping{$chunk_key};
300 delete $target_file_key_to_line_mapping{$chunk_key};
301 }
302 }
303
304 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
305 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
306 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
307 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
308
309 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
310 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
311
312
313 # Chunks needing updating are those in the target file that have been more recently edited in the source file
314 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
315 my @uptodate_target_file_keys = ();
316 foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
317 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
318 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
319
320 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
321
322 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
323 # &log_message("Chunk with key $chunk_key needs updating.");
324 push(@uptodate_target_file_keys, $chunk_key);
325 }
326 }
327
328 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
329
330 return $xml_response;
331}
332
333
334sub get_first_n_chunks_requiring_work
335{
336 # The code of the target language (ensure it is lowercase)
337 my $target_language_code = lc(shift(@_));
338 # The key of the file to translate (ensure it is lowercase)
339 my $translation_file_key = lc(shift(@_));
340 # The number of chunks to return (defaults to one if not specified)
341 my $num_chunks_to_return = shift(@_) || "1";
342
343 # Check that the necessary arguments were supplied
344 if (!$target_language_code || !$translation_file_key) {
345 &throw_fatal_error("Missing command argument.");
346 }
347
348 # Get (and check) the translation configuration
349 my ($source_file, $target_file, $translation_file_type)
350 = &get_translation_configuration($target_language_code, $translation_file_key);
351
352 # Parse the source language and target language files
353 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
354 my @source_file_lines = &read_file_lines($source_file_path);
355 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
356
357 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
358 my @target_file_lines = &read_file_lines($target_file_path);
359 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
360
361 # Filter out any automatically translated chunks
362 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
363 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
364 delete $source_file_key_to_line_mapping{$chunk_key};
365 delete $target_file_key_to_line_mapping{$chunk_key};
366 }
367 }
368
369 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
370 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
371 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
372 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
373
374 # Determine the target file chunks requiring translation
375 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
376 &log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
377
378 # Determine the target file chunks requiring updating
379 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
380 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
381 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
382 &log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
383
384 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
385
386 return $xml_response;
387}
388
389
390sub get_language_status
391{
392 # The code of the target language (ensure it is lowercase)
393 my $target_language_code = lc(shift(@_));
394
395 # Check that the necessary arguments were supplied
396 if (!$target_language_code) {
397 &throw_fatal_error("Missing command argument.");
398 }
399
400 # Form an XML response to the command
401 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
402 $xml_response .= "<GTIResponse>\n";
403 $xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
404
405 foreach my $translation_file (@$gti_translation_files) {
406 my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
407 my $target_file_name = "";
408
409 if ($translation_file->{'key'} =~ m/^gs3/) { # gs3interface, gs3colcfg
410 my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
411 &build_gs3_configuration($translation_file->{'key'}, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
412
413 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
414 my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
415
416 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
417 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
418 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
419 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
420 }
421 else {
422 # Get (and check) the translation configuration
423 my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
424 $target_file_name = $target_file;
425
426 # Parse the source language and target language files
427 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
428 my @source_file_lines = &read_file_lines($source_file_path);
429 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
430
431 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
432 my @target_file_lines = &read_file_lines($target_file_path);
433 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
434
435 # Filter out any automatically translated chunks
436 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
437 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
438 delete $source_file_key_to_line_mapping{$chunk_key};
439 delete $target_file_key_to_line_mapping{$chunk_key};
440 }
441 }
442
443 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
444 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
445
446 # Determine the target file chunks requiring translation
447 my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
448
449 # Determine the target file chunks requiring updating
450 my @target_file_keys_requiring_updating = ();
451 if (-e $target_file_path) {
452 my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
453 my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
454 @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
455 }
456
457 $num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
458 $num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
459 $num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
460 $num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
461 }
462
463 &log_message("Status of " . $translation_file->{'key'});
464 &log_message("Number of source chunks: " . $num_source_chunks);
465 &log_message("Number of target chunks: " . $num_target_chunks);
466 &log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
467 &log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
468
469 $xml_response .= " <TranslationFile"
470 . " key=\"" . $translation_file->{'key'} . "\""
471 . " target_file_path=\"" . $target_file_name . "\""
472 . " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
473 . " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
474 . " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
475 }
476
477 $xml_response .= " </LanguageStatus>\n";
478
479 $xml_response .= "</GTIResponse>\n";
480 return $xml_response;
481}
482
483
484sub search_chunks
485{
486 # The code of the target language (ensure it is lowercase)
487 my $target_language_code = lc(shift(@_));
488 # The key of the file to translate (ensure it is lowercase)
489 my $translation_file_key = lc(shift(@_));
490 # The query string
491 my $query_string = join(' ', @_);
492
493 # Check that the necessary arguments were supplied
494 if (!$target_language_code || !$translation_file_key || !$query_string) {
495 &throw_fatal_error("Missing command argument.");
496 }
497
498 my ($source_file, $target_file, $translation_file_type) = ();
499 my %source_file_key_to_text_mapping = ();
500 my %target_file_key_to_text_mapping = ();
501
502
503 if ($translation_file_key !~ m/^gs3/) {
504 # Get (and check) the translation configuration
505 ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
506
507 # Parse the source language and target language files
508 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
509 my @source_file_lines = &read_file_lines($source_file_path);
510 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
511
512 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
513 my @target_file_lines = &read_file_lines($target_file_path);
514 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
515
516 # Filter out any automatically translated chunks
517 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
518 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
519 delete $source_file_key_to_line_mapping{$chunk_key};
520 delete $target_file_key_to_line_mapping{$chunk_key};
521 }
522 }
523
524 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
525 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
526 }
527 else {
528 # Not needed in this case
529 my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
530 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
531 \%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping, 1);
532 }
533
534 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
535 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
536
537 # Determine the target file chunks matching the query
538 my @target_file_keys_matching_query = ();
539 foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
540 my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
541 if ($target_file_text =~ /$query_string/i) {
542 # &log_message("Chunk with key $chunk_key matches query.");
543 push(@target_file_keys_matching_query, $chunk_key);
544 }
545 }
546
547 # Form an XML response to the command
548 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
549 $xml_response .= "<GTIResponse>\n";
550
551 $xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
552 foreach my $chunk_key (@target_file_keys_matching_query) {
553 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
554
555 $xml_response .= " <Chunk key=\"$chunk_key\">\n";
556 $xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
557 $xml_response .= " </Chunk>\n";
558 }
559 $xml_response .= " </ChunksMatchingQuery>\n";
560
561 $xml_response .= "</GTIResponse>\n";
562 return $xml_response;
563}
564
565
566sub submit_translations
567{
568 # The code of the target language (ensure it is lowercase)
569 my $target_language_code = lc(shift(@_));
570 # The key of the file to translate (ensure it is lowercase)
571 my $translation_file_key = lc(shift(@_));
572 # The username of the translation submitter
573 my $submitter_username = shift(@_);
574 # Whether to submit a target chunk even if it hasn't changed
575 my $force_submission_flag = shift(@_);
576
577 # Check that the necessary arguments were supplied
578 if (!$target_language_code || !$translation_file_key || !$submitter_username) {
579 &log_message("Fatal error (but cannot be thrown): Missing command argument.");
580 die "\n";
581 }
582
583 my %source_file_key_to_text_mapping = ();
584 my %source_file_key_to_gti_comment_mapping = ();
585 my %target_file_key_to_text_mapping = ();
586 my %target_file_key_to_gti_comment_mapping = ();
587
588 my (@source_file_lines, @target_file_lines) = ();
589 my ($source_file, $target_file, $translation_file_type);
590
591
592 if ($translation_file_key !~ m/^gs3/) {
593 # Get (and check) the translation configuration
594 ($source_file, $target_file, $translation_file_type)
595 = &get_translation_configuration($target_language_code, $translation_file_key);
596
597 # Parse the source language and target language files
598 @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
599 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
600 %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
601 %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
602
603 @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
604 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
605 %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
606 %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
607 }
608 else {
609 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
610 \%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping, 1);
611 }
612 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
613 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
614
615 # Submission date
616 my $day = (localtime)[3];
617 my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
618 my $year = (localtime)[5] + 1900;
619 my $submission_date = "$day-$month-$year";
620
621 open(SUBMISSION, "-");
622 my @submission_lines = <SUBMISSION>;
623 close(SUBMISSION);
624
625 # Remove any nasty carriage returns
626 # &log_message("Submission:");
627 foreach my $submission_line (@submission_lines) {
628 $submission_line =~ s/\r$//;
629 #&log_message(" $submission_line");
630 }
631
632 my %source_file_key_to_submission_mapping = ();
633 my %target_file_key_to_submission_mapping = ();
634 for (my $i = 0; $i < scalar(@submission_lines); $i++) {
635 # Read source file part of submission
636 if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
637 my $chunk_key = $1;
638
639 # Read the source file text
640 my $source_file_chunk_text = "";
641 $i++;
642 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
643 $source_file_chunk_text .= $submission_lines[$i];
644 $i++;
645 }
646 $source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
647 $source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
648
649 #&log_message("Source file key: $chunk_key");
650 #&log_message("Source file text: $source_file_chunk_text");
651 $source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
652 }
653
654 # Read target file part of submission
655 if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
656 my $chunk_key = $1;
657
658 # Read the target file text
659 my $target_file_chunk_text = "";
660 $i++;
661 while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
662 $target_file_chunk_text .= $submission_lines[$i];
663 $i++;
664 }
665 $target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
666 $target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
667
668 #&log_message("Target file key: $chunk_key");
669 #&log_message("Target file text: $target_file_chunk_text");
670 $target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
671 }
672 }
673
674 # -----------------------------------------
675 # Validate the translation submissions
676 # -----------------------------------------
677
678 # Check that the translations are valid
679 foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
680 # Make sure the submitted chunk still exists in the source file
681 if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
682 &log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
683 delete $source_file_key_to_submission_mapping{$chunk_key};
684 delete $target_file_key_to_submission_mapping{$chunk_key};
685 next;
686 }
687
688 # Make sure the submitted source chunk matches the source file chunk
689 if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
690 #if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
691 #print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
692 #print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
693
694 &log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
695 &log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
696 &log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
697 delete $source_file_key_to_submission_mapping{$chunk_key};
698 delete $target_file_key_to_submission_mapping{$chunk_key};
699 next;
700 }
701 }
702
703 # Apply the submitted translations
704 foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
705 # Only apply the submission if it is a change, unless -force_submission has been specified
706 if ($force_submission_flag || !defined($target_file_key_to_text_mapping{$chunk_key}) || $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
707 $target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
708 $target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
709 }
710 }
711
712 if ($translation_file_key !~ m/^gs3/) {
713 eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
714 } else {
715 eval "&write_translated_gs3interface(\$translation_file_key, \\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
716 }
717}
718
719
720sub create_glihelp_zip_file
721{
722 my $target_language_code = shift(@_);
723 my $translation_file_key = "glihelp";
724
725 &log_message("Creating GLI Help zip file for $target_language_code");
726
727 my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
728
729 my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
730 my $oldclasspath = $classpath;
731 if ( ! -e $classpath) {
732 $classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
733 }
734 if ( ! -e $classpath) {
735 &throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
736 }
737
738
739 my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
740 my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
741 my $os = $^O;
742 my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
743 my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
744 $classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
745
746 my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
747 $gli_help_directory = &util::filename_cat($gli_help_directory, "help");
748
749 my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
750 if ( ! -e $gen_many_html_xsl_filepath) {
751 &throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
752 }
753
754 my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
755 my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
756
757 my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
758 $target_file_directory = $target_file_directory."/";
759
760 my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
761
762 # if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
763 my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
764 if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
765
766 # copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
767 my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
768 my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
769 &FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
770
771 # In gli/help/<newlang>/help.xml, replace all occurrences of
772 # <Text id="1">This text in en will be removed for new langcode</Text>
773 # with <!-- Missing translation: 1 -->
774 open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
775 my $help_xml_contents;
776 # Read in the entire contents of the file in one hit
777 sysread(FIN, $help_xml_contents, -s FIN);
778 close(FIN);
779
780 $help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
781
782 open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
783 print FOUT $help_xml_contents;
784 close(FOUT);
785 }
786
787 my $perl_exec = &util::get_perl_exec();
788 my $java_exec = "java";
789 if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
790 $java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
791 }
792
793 #my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
794 my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath | \"$perl_exec\" -S $split_script_filepath $target_file_directory";
795 #&throw_fatal_error("RAN gti command: $cmd");
796 my $response = `$cmd`;
797
798 #$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
799 $cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
800 $response = `$cmd`;
801
802 # create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
803 my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
804 if(!&FileUtils::directoryExists($tmpdir)) {
805 &FileUtils::makeDirectory($tmpdir);
806 }
807 #my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
808 my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
809 $cmd = "zip -rj $zip_file_path $target_file_directory -i \*.htm \*.xml";
810
811 $response = `$cmd`;
812}
813
814
815sub get_translation_configuration
816{
817 # Get the code of the target language
818 my $target_language_code = shift(@_);
819 # Get the key of the file to translate
820 my $translation_file_key = shift(@_);
821
822 # Read the translation data from the gti.cfg file
823 my ($source_file, $target_file, $translation_file_type) =
824 &get_translation_data_for($target_language_code, $translation_file_key);
825
826 # Check that the file to translate is defined in the gti.cfg file
827 if (!$source_file || !$target_file || !$translation_file_type) {
828 &throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
829 }
830
831 # Check that the source file exists
832 my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
833 if (!-e $source_file_path) {
834 &throw_fatal_error("Source file $source_file_path does not exist.");
835 }
836
837 # Check that the source file is up to date
838 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
839 # unless ($translation_file_is_not_in_cvs) {
840 #my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
841 my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
842 if ($source_file_cvs_status =~ /^C /) {
843 &throw_fatal_error("Source file $source_file_path conflicts with the repository.");
844 }
845 if ($source_file_cvs_status =~ /^M /) {
846 &throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
847 }
848 # }
849
850 return ($source_file, $target_file, $translation_file_type);
851}
852
853
854sub get_translation_data_for
855{
856 my ($target_language_code, $translation_file_key) = @_;
857
858 foreach my $translation_file (@$gti_translation_files) {
859 # If this isn't the correct translation file, move onto the next one
860 next if ($translation_file_key ne $translation_file->{'key'});
861
862 # Resolve the target language file
863 my $target_language_file = $translation_file->{'target_file'};
864 if ($target_language_file =~ /(\{.+\;.+\})/) {
865 my $unresolved_target_language_file_part = $1;
866
867 # Check for a special case for the target language code
868 if ($unresolved_target_language_file_part =~ /(\{|\;)$target_language_code:([^\;]+)(\;|\})/) {
869 my $resolved_target_language_file_part = $2;
870 $target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
871 }
872 # Otherwise use the last part as the default value
873 else {
874 my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
875 $target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
876 }
877 }
878
879 # Resolve instances of {iso_639_1_target_language_name}
880 my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
881 $iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
882 $target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
883
884 # Resolve instances of {target_language_code}
885 $target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
886
887 return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
888}
889
890return ();
891}
892
893
894sub read_file_lines
895{
896 my ($file_path) = @_;
897
898 if (!open(FILE_IN, "<$file_path")) {
899 &log_message("Note: Could not open file $file_path.");
900 return ();
901 }
902 my @file_lines = <FILE_IN>;
903 close(FILE_IN);
904
905 return @file_lines;
906}
907
908
909sub build_key_to_line_mapping
910{
911 my ($file_lines, $translation_file_type) = @_;
912 eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
913}
914
915
916sub build_key_to_text_mapping
917{
918 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
919
920 &log_message("@@@ HELLO WORLD");
921
922
923 my %key_to_text_mapping = ();
924 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
925 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
926 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
927
928 my $chunk_text = @$file_lines[$chunk_starting_line];
929 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
930 $chunk_text .= @$file_lines[$l];
931 }
932
933 # Map from chunk key to text
934 eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
935
936 #if($chunk_key =~ m/document\\/) {
937 #&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
938 #&log_message("@@@ chunk key: $chunk_key");
939 #}
940
941 }
942
943 return %key_to_text_mapping;
944}
945
946
947sub build_key_to_last_update_date_mapping
948{
949 my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
950
951 # If the files aren't in CVS then we can't tell anything about what needs updating
952 # return () if ($translation_file_is_not_in_cvs);
953
954 # Build a mapping from key to CVS date
955 # Need to be careful with this mapping because the chunk keys won't necessarily all be valid
956 my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
957
958 # Build a mapping from key to comment date
959 my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
960
961 # Build a mapping from key to last update date (the latter of the CVS date and comment date)
962 my %key_to_last_update_date_mapping = ();
963 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
964 # Use the CVS date as a starting point
965 my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
966 $key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
967
968 # If a comment date exists and it is after the CVS date, use that instead
969 # need to convert the comment date format to SVN format
970 my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
971 if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
972 my $chunk_comment_date = $1;
973 if ((!defined($chunk_cvs_date) || &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
974 $key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
975 }
976 }
977 }
978
979 return %key_to_last_update_date_mapping;
980}
981
982
983sub build_key_to_cvs_date_mapping
984{
985 my ($filename, $translation_file_type) = @_;
986
987 # Use SVN to annotate each line of the file with the date it was last edited
988 # The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
989 my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
990
991 my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
992
993 my @cvs_annotated_file_lines_date = ();
994 foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
995 # Extract the date from the SVN annotation at the front
996 # svn format : 2007-07-16
997 $cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
998
999 push(@cvs_annotated_file_lines_date, $1);
1000
1001 # trim extra date information in svn annotation format
1002 # 15:42:49 +1200 (Wed, 21 Jun 2006)
1003 $cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
1004 }
1005
1006 # Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
1007 my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
1008
1009 my %key_to_cvs_date_mapping = ();
1010 foreach my $chunk_key (keys(%key_to_line_mapping)) {
1011 my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
1012 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
1013
1014 # Find the date this chunk was last edited, from the CVS annotation
1015 my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
1016 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1017 if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
1018 # This part of the chunk has been updated more recently
1019 $chunk_date = $cvs_annotated_file_lines_date[$l];
1020
1021 }
1022 }
1023
1024 # Map from chunk key to CVS date
1025 $key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
1026 }
1027
1028 return %key_to_cvs_date_mapping;
1029}
1030
1031
1032sub build_key_to_gti_comment_mapping
1033{
1034 my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
1035
1036 my %key_to_gti_comment_mapping = ();
1037 foreach my $chunk_key (keys(%$key_to_line_mapping)) {
1038 my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
1039 my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
1040
1041 my $chunk_text = @$file_lines[$chunk_starting_line];
1042 for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
1043 $chunk_text .= @$file_lines[$l];
1044 }
1045
1046 # Map from chunk key to GTI comment
1047 my $chunk_gti_comment;
1048 eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
1049 $key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
1050 }
1051
1052 return %key_to_gti_comment_mapping;
1053}
1054
1055
1056sub determine_chunks_requiring_translation
1057{
1058 my $source_file_key_to_text_mapping = shift(@_);
1059 my $target_file_key_to_text_mapping = shift(@_);
1060
1061 # Chunks needing translation are those in the source file with no translation in the target file
1062 my @target_file_keys_requiring_translation = ();
1063 foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
1064 if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
1065 # &log_message("Chunk with key $chunk_key needs translating.");
1066 push(@target_file_keys_requiring_translation, $chunk_key);
1067 }
1068 }
1069
1070 return @target_file_keys_requiring_translation;
1071}
1072
1073
1074sub determine_chunks_requiring_updating
1075{
1076 my $source_file_key_to_last_update_date_mapping = shift(@_);
1077 my $target_file_key_to_last_update_date_mapping = shift(@_);
1078
1079 # Chunks needing updating are those in the target file that have been more recently edited in the source file
1080 my @target_file_keys_requiring_updating = ();
1081 foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
1082 my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
1083 my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
1084
1085 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
1086
1087 if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
1088 # &log_message("Chunk with key $chunk_key needs updating.");
1089 # &log_message("key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n");
1090 push(@target_file_keys_requiring_updating, $chunk_key);
1091 }
1092 }
1093
1094 return @target_file_keys_requiring_updating;
1095}
1096
1097
1098sub is_chunk_automatically_translated
1099{
1100 my ($chunk_key, $translation_file_type) = @_;
1101 eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
1102}
1103
1104
1105sub make_text_xml_safe
1106{
1107 my $text = shift(@_);
1108 $text =~ s/\&/\&amp\;/g;
1109 $text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
1110 $text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
1111 $text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
1112 $text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
1113 $text =~ s/</\&lt\;/g;
1114 $text =~ s/>/\&gt\;/g;
1115 return $text;
1116}
1117
1118
1119sub unmake_text_xml_safe
1120{
1121 my $text = shift(@_);
1122 $text =~ s/\&lt\;/</g;
1123 $text =~ s/\&gt\;/>/g;
1124 $text =~ s/\&amp\;/\&/g;
1125 return $text;
1126}
1127
1128
1129# Returns 1 if $date1 is after $date2, 0 otherwise
1130sub is_date_after_cvs
1131{
1132 my ($date1, $date2) = @_;
1133 my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
1134 "Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
1135
1136 if(!defined $date1) {
1137 return 1;
1138 }
1139
1140 my @date1parts = split(/-/, $date1);
1141 my @date2parts = split(/-/, $date2);
1142
1143 # Compare year - nasty because we have rolled over into a new century
1144 my $year1 = $date1parts[2];
1145 if ($year1 < 80) {
1146 $year1 += 2000;
1147 }
1148 my $year2 = $date2parts[2];
1149 if ($year2 < 80) {
1150 $year2 += 2000;
1151 }
1152
1153 # Compare year
1154 if ($year1 > $year2) {
1155 return 1;
1156 }
1157 elsif ($year1 == $year2) {
1158 # Year is the same, so compare month
1159 if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
1160 return 1;
1161 }
1162 elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
1163 # Month is the same, so compare day
1164 if ($date1parts[0] > $date2parts[0]) {
1165 return 1;
1166 }
1167 }
1168 }
1169
1170 return 0;
1171}
1172
1173sub is_date_after
1174{
1175 my ($date1, $date2) = @_;
1176
1177 if(!defined $date1) {
1178 return 1;
1179 }
1180 if(!defined $date2) {
1181 return 0;
1182 }
1183
1184 # 16-Aug-2006
1185 if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1186 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1187 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1188 $date1=$3 . "-" . $months{$2} . "-" . $1;
1189 # print "** converted date1: $date1\n";
1190 }
1191 if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
1192 my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
1193 "Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
1194 $date2=$3 . "-" . $months{$2} . "-" . $1;
1195 # print "** converted date2: $date2\n";
1196 }
1197
1198
1199 # 2006-08-16
1200 my @date1parts = split(/-/, $date1);
1201 my @date2parts = split(/-/, $date2);
1202
1203 # Compare year
1204 if ($date1parts[0] > $date2parts[0]) {
1205 return 1;
1206 }
1207 elsif ($date1parts[0] == $date2parts[0]) {
1208 # Year is the same, so compare month
1209 if ($date1parts[1] > $date2parts[1]) {
1210 return 1;
1211 }
1212 elsif ($date1parts[1] == $date2parts[1]) {
1213 # Month is the same, so compare day
1214 if ($date1parts[2] > $date2parts[2]) {
1215 return 1;
1216 }
1217 }
1218 }
1219
1220 return 0;
1221}
1222
1223
1224sub create_xml_response_for_chunks_requiring_work
1225{
1226 my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1227
1228 # Form an XML response to the command
1229 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1230 $xml_response .= "<GTIResponse>\n";
1231 $xml_response .= " <TranslationFile"
1232 . " key=\"" . $translation_file_key . "\""
1233 . " target_file_path=\"" . $target_file . "\""
1234 . " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
1235 . " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
1236 . " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
1237
1238 # Do chunks requiring translation first
1239 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
1240 $xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
1241 }
1242 else {
1243 $xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
1244 }
1245
1246 my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
1247 foreach my $chunk_key (@sorted_chunk_keys) {
1248 last if ($num_chunks_to_return == 0);
1249
1250 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1251 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1252
1253 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1254 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1255 $xml_response .= " <TargetFileText></TargetFileText>\n";
1256 $xml_response .= " </Chunk>\n";
1257
1258 $num_chunks_to_return--;
1259 }
1260
1261 $xml_response .= " </ChunksRequiringTranslation>\n";
1262
1263 # Then do chunks requiring updating
1264 if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
1265 $xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
1266 }
1267 else {
1268 $xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
1269 }
1270
1271 # foreach my $chunk_key (@target_file_keys_requiring_updating) {
1272 @sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
1273 foreach my $chunk_key (@sorted_chunk_keys) {
1274 last if ($num_chunks_to_return == 0);
1275
1276 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1277 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1278 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1279 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1280
1281 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1282 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1283 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1284 $xml_response .= " </Chunk>\n";
1285
1286 $num_chunks_to_return--;
1287 }
1288
1289 $xml_response .= " </ChunksRequiringUpdating>\n";
1290
1291 $xml_response .= "</GTIResponse>\n";
1292
1293 return $xml_response;
1294}
1295
1296sub create_xml_response_for_uptodate_chunks
1297{
1298 my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
1299
1300 # Form an XML response to the command
1301 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1302 $xml_response .= "<GTIResponse>\n";
1303 $xml_response .= " <TranslationFile"
1304 . " key=\"" . $translation_file_key . "\""
1305 . " target_file_path=\"" . $target_file . "\""
1306 . " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
1307
1308
1309 # Then do chunks requiring updating
1310 $xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
1311
1312
1313 # foreach my $chunk_key (@uptodate_target_file_keys) {
1314 my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
1315 foreach my $chunk_key (@sorted_chunk_keys) {
1316
1317 my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1318 my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
1319 my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} || "";
1320 my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
1321
1322 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1323 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1324 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1325 $xml_response .= " </Chunk>\n";
1326
1327 }
1328
1329 $xml_response .= " </UptodateChunks>\n";
1330
1331 $xml_response .= "</GTIResponse>\n";
1332
1333 return $xml_response;
1334}
1335
1336sub create_xml_response_for_all_chunks
1337{
1338 my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
1339
1340 # Form an XML response to the command
1341 my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
1342 $xml_response .= "<GTIResponse>\n";
1343 $xml_response .= " <TranslationFile"
1344 . " key=\"" . $translation_file_key . "\""
1345 . " target_file_path=\"" . $target_file . "\"\/>\n";
1346
1347 # Do all the chunks
1348 $xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
1349
1350 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
1351 foreach my $chunk_key (@sorted_chunk_keys) {
1352 my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1353 my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
1354
1355 $xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
1356 $xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
1357 if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
1358 my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} || "";
1359 my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
1360 $xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
1361 }
1362 else {
1363 $xml_response .= " <TargetFileText></TargetFileText>\n";
1364 }
1365
1366 $xml_response .= " </Chunk>\n";
1367 }
1368 $xml_response .= " </Chunks>\n";
1369
1370 $xml_response .= "</GTIResponse>\n";
1371 return $xml_response;
1372}
1373
1374
1375
1376# ==========================================================================================
1377# MACROFILE FUNCTIONS
1378
1379sub build_key_to_line_mapping_for_macrofile
1380{
1381 my (@file_lines) = @_;
1382
1383 my $macro_package;
1384 my %chunk_key_to_line_mapping = ();
1385 # Process the contents of the file, line by line
1386 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1387 my $line = $file_lines[$i];
1388 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1389
1390 # Check if a new package is being defined
1391 if ($line =~ m/^package\s+(.+)/) {
1392 $macro_package = $1;
1393 }
1394
1395 # Line contains a macro name
1396 elsif ($line =~ m/^(_\w+_)/) {
1397 my $macro_key = $1;
1398 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1399
1400 # While there is still text of the macro to go...
1401 my $startline = $i;
1402 while ($line !~ /\}$/) {
1403 $i++;
1404 if ($i == scalar(@file_lines)) {
1405 &throw_fatal_error("Could not find end of macro $macro_key.");
1406 }
1407 $line = $file_lines[$i];
1408 $line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
1409 }
1410
1411 # The chunk key consists of the package name and the macro key
1412 my $chunk_key = $macro_package . "." . $macro_key;
1413 # Map from chunk key to line
1414 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1415 }
1416
1417 # Icon: line in format ## "image text" ## image_type ## macro_name ##
1418 elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
1419 # The chunk key consists of package name and macro key
1420 my $chunk_key = $macro_package . "." . $1;
1421 # Map from chunk key to line
1422 $chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
1423}
1424}
1425
1426return %chunk_key_to_line_mapping;
1427}
1428
1429
1430sub import_chunk_from_macrofile
1431{
1432 my ($chunk_text) = @_;
1433
1434 # Is this an icon macro??
1435 if ($chunk_text =~ /^\#\# (.*)/) {
1436 # Extract image macro text
1437 $chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
1438 $chunk_text = $1;
1439
1440 # Remove enclosing quotes
1441 $chunk_text =~ s/^\"//;
1442 $chunk_text =~ s/\"$//;
1443 }
1444
1445 # No, so it must be a text macro
1446 else {
1447 # Remove macro key
1448 $chunk_text =~ s/^_([^_]+)_(\s*)//;
1449
1450 # Remove language specifier
1451 $chunk_text =~ s/^\[l=[^\]]*\](\s*)//; # only remove until first closing square bracket, ]
1452
1453 # Remove braces enclosing text
1454 $chunk_text =~ s/^{(\s*)((.|\n)*)}(\s*)(\#.+\s*)?/$2/;
1455 }
1456
1457 return $chunk_text;
1458}
1459
1460
1461sub get_macrofile_chunk_gti_comment
1462{
1463 my ($chunk_text) = @_;
1464
1465 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1466 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1467 return $1;
1468}
1469
1470return undef;
1471}
1472
1473
1474sub is_macrofile_chunk_automatically_translated
1475{
1476 my ($chunk_key) = @_;
1477
1478 # The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
1479 if ($chunk_key =~ /\._(httpicon|width|height)/) {
1480 return 1;
1481 }
1482
1483 return 0;
1484}
1485
1486
1487# Use the source file to generate a target file that is formatted the same
1488sub write_translated_macrofile
1489{
1490 my $source_file = shift(@_); # Not used
1491 my @source_file_lines = @{shift(@_)};
1492 my $source_file_key_to_text_mapping = shift(@_);
1493 my $target_file = shift(@_);
1494 my @target_file_lines = @{shift(@_)};
1495 my $target_file_key_to_text_mapping = shift(@_);
1496 my $target_file_key_to_gti_comment_mapping = shift(@_);
1497 my $target_language_code = shift(@_);
1498
1499 # Build a mapping from source file line to chunk key
1500 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
1501 my %source_file_line_to_key_mapping = ();
1502 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1503 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1504 }
1505 my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
1506 my $source_file_line_number = 0;
1507
1508 # Build a mapping from target file line to chunk key
1509 my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
1510 my %target_file_line_to_key_mapping = ();
1511 foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
1512 $target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1513 }
1514 my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
1515
1516 # Write the new target file
1517 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1518 if (!open(TARGET_FILE, ">$target_file_path")) {
1519 &throw_fatal_error("Could not write target file $target_file_path.");
1520 }
1521
1522 # Use the header from the target file, to keep language and author information
1523 if (scalar(@target_file_line_keys) > 0) {
1524 my $target_file_line_number = 0;
1525 my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
1526 while ($target_file_line_number < $target_file_chunk_starting_line_number) {
1527 my $target_file_line = $target_file_lines[$target_file_line_number];
1528 last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
1529 print TARGET_FILE $target_file_line;
1530 $target_file_line_number++;
1531 }
1532
1533 $source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
1534 }
1535
1536 # Model the new target file on the source file, with the target file translations
1537 foreach my $line_key (@source_file_line_keys) {
1538 # Fill in the gaps before this chunk starts
1539 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1540 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1541 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1542 print TARGET_FILE $source_file_lines[$source_file_line_number];
1543 $source_file_line_number++;
1544 }
1545 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1546
1547 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1548 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1549 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1550
1551 my $macrofile_key = $chunk_key;
1552 $macrofile_key =~ s/^(.+?)\.//;
1553
1554 # If no translation exists for this chunk, show this, and move on
1555 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1556 print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
1557 next;
1558 }
1559
1560 # Grab the source chunk text
1561 my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
1562 for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
1563 $source_file_chunk .= $source_file_lines[$l];
1564 }
1565
1566 # Is this an icon macro??
1567 if ($source_file_chunk =~ /^\#\# (.*)/) {
1568 # Escape any newline and question mark characters so the source text is replaced correctly
1569 $source_file_chunk_text =~ s/\\/\\\\/g;
1570 $source_file_chunk_text =~ s/\?/\\\?/g;
1571
1572 # Build the new target chunk from the source chunk
1573 my $target_file_chunk = $source_file_chunk;
1574 $target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
1575 $target_file_chunk =~ s/(\s)*$//;
1576 print TARGET_FILE "$target_file_chunk";
1577 }
1578
1579 # No, it is just a normal text macro
1580 else {
1581 print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
1582 }
1583
1584 # Add the "updated" comment, if one exists
1585 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1586 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1587 }
1588 print TARGET_FILE "\n";
1589}
1590
1591close(TARGET_FILE);
1592}
1593
1594
1595sub sort_by_line
1596{
1597 return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
1598}
1599
1600
1601# ==========================================================================================
1602# RESOURCE BUNDLE FUNCTIONS
1603
1604# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
1605sub build_key_to_line_mapping_for_resource_bundle
1606{
1607 my (@file_lines) = @_;
1608
1609 my %chunk_key_to_line_mapping = ();
1610
1611 my $chunk_key;
1612 my $startindex = -1;
1613
1614 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1615 my $line = $file_lines[$i];
1616 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1617
1618 # Line contains a dictionary string
1619 if ($line =~ /^(\S+)[:|=](.*)$/) {
1620 $chunk_key = $1;
1621 $startindex = $i;
1622 }
1623 if ($startindex != -1) {
1624 if($line !~ m/\\$/) { # line finished
1625 # $i keeps track of the line at which this property (chunk_key) finishes
1626
1627 # Map from chunk key to line
1628 $chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
1629 $startindex = -1;
1630 $chunk_key = "";
1631 }
1632 }
1633 }
1634
1635 return %chunk_key_to_line_mapping;
1636}
1637
1638
1639sub import_chunk_from_resource_bundle
1640{
1641 my ($chunk_text) = @_;
1642
1643 # Simple: just remove string key
1644 $chunk_text =~ s/^(\S+)[:|=](\s*)//s;
1645 $chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
1646 $chunk_text =~ s/(\s*)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.*)\s*$//is;
1647
1648 return $chunk_text;
1649}
1650
1651
1652sub get_resource_bundle_chunk_gti_comment
1653{
1654 my ($chunk_text) = @_;
1655
1656 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1657 if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.*)\s*$/i) {
1658 return $1;
1659}
1660
1661return undef;
1662}
1663
1664
1665sub is_resource_bundle_chunk_automatically_translated
1666{
1667 # No resource bundle chunks are automatically translated
1668 return 0;
1669}
1670
1671
1672sub write_translated_resource_bundle
1673{
1674 my $source_file = shift(@_); # Not used
1675 my @source_file_lines = @{shift(@_)};
1676 my $source_file_key_to_text_mapping = shift(@_);
1677 my $target_file = shift(@_);
1678 my @target_file_lines = @{shift(@_)}; # Not used
1679 my $target_file_key_to_text_mapping = shift(@_);
1680 my $target_file_key_to_gti_comment_mapping = shift(@_);
1681 my $target_language_code = shift(@_); # Not used
1682
1683 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1684 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
1685 my %source_file_line_to_key_mapping = ();
1686 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1687 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1688 }
1689
1690 # Write the new target file
1691 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1692 if (!open(TARGET_FILE, ">$target_file_path")) {
1693 &throw_fatal_error("Could not write target file $target_file_path.");
1694 }
1695
1696 # Model the new target file on the source file, with the target file translations
1697 my $source_file_line_number = 0;
1698 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1699 # Fill in the gaps before this chunk starts
1700 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1701 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1702 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1703 print TARGET_FILE $source_file_lines[$source_file_line_number];
1704 $source_file_line_number++;
1705 }
1706 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1707
1708 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1709 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1710 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1711
1712 # If no translation exists for this chunk, show this, and move on
1713 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1714 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
1715 next;
1716 }
1717
1718 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
1719 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1720 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
1721 }
1722 print TARGET_FILE "\n";
1723 }
1724
1725 close(TARGET_FILE);
1726}
1727
1728
1729# ==========================================================================================
1730# GREENSTONE XML FUNCTIONS
1731
1732sub build_key_to_line_mapping_for_greenstone_xml
1733{
1734 my (@file_lines) = @_;
1735
1736 my %chunk_key_to_line_mapping = ();
1737 for (my $i = 0; $i < scalar(@file_lines); $i++) {
1738 my $line = $file_lines[$i];
1739 $line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
1740
1741 # Line contains a string to translate
1742 if ($line =~ /^\s*<Text id=\"(.*?)\">/) {
1743 my $chunk_key = $1;
1744 $line =~ s/\s*$//; # Remove any nasty whitespace
1745 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1746
1747 # While there is still text of the string to go...
1748 my $startline = $i;
1749 while ($line !~ /<\/Text>$/) {
1750 $i++;
1751 if ($i == scalar(@file_lines)) {
1752 &throw_fatal_error("Could not find end of string $chunk_key.");
1753 }
1754 $line = $file_lines[$i];
1755 $line =~ s/\s*$//; # Remove any nasty whitespace
1756 $line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1757 }
1758
1759 # Map from chunk key to line
1760 if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
1761 $chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
1762 }
1763 else {
1764 &throw_fatal_error("Duplicate key $chunk_key.");
1765 }
1766 }
1767 }
1768
1769 return %chunk_key_to_line_mapping;
1770}
1771
1772
1773sub import_chunk_from_greenstone_xml
1774{
1775 my ($chunk_text) = @_;
1776
1777 # Simple: just remove the Text tags
1778 $chunk_text =~ s/^\s*<Text id=\"(.*?)\">(\s*)//;
1779 $chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
1780 $chunk_text =~ s/<\/Text>$//;
1781
1782 return $chunk_text;
1783}
1784
1785
1786sub get_greenstone_xml_chunk_gti_comment
1787{
1788 my ($chunk_text) = @_;
1789
1790 # Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
1791 if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
1792 return $1;
1793 }
1794
1795 return undef;
1796}
1797
1798
1799sub is_greenstone_xml_chunk_automatically_translated
1800{
1801 # No greenstone XML chunks are automatically translated
1802 return 0;
1803}
1804
1805
1806sub write_translated_greenstone_xml
1807{
1808 my $source_file = shift(@_); # Not used
1809 my @source_file_lines = @{shift(@_)};
1810 my $source_file_key_to_text_mapping = shift(@_);
1811 my $target_file = shift(@_);
1812 my @target_file_lines = @{shift(@_)}; # Not used
1813 my $target_file_key_to_text_mapping = shift(@_);
1814 my $target_file_key_to_gti_comment_mapping = shift(@_);
1815 my $target_language_code = shift(@_); # Not used
1816
1817 # Build a mapping from chunk key to source file line, and from source file line to chunk key
1818 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
1819 my %source_file_line_to_key_mapping = ();
1820 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
1821 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
1822 }
1823
1824 # Write the new target file
1825 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
1826 if (!open(TARGET_FILE, ">$target_file_path")) {
1827 &throw_fatal_error("Could not write target file $target_file_path.");
1828 }
1829
1830 # Model the new target file on the source file, with the target file translations
1831 my $source_file_line_number = 0;
1832 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
1833 # Fill in the gaps before this chunk starts
1834 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
1835 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
1836 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
1837 print TARGET_FILE $source_file_lines[$source_file_line_number];
1838 $source_file_line_number++;
1839 }
1840 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
1841
1842 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
1843 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
1844 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} || "";
1845 $target_file_chunk_text =~ s/(\n)*$//g;
1846
1847 # If no translation exists for this chunk, show this, and move on
1848 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
1849 print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
1850 next;
1851 }
1852
1853 print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
1854 if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
1855 my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
1856 $chunk_gti_comment =~ s/^Updated //;
1857 print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
1858 }
1859 print TARGET_FILE "\n";
1860 }
1861
1862 # Fill in the end of the file
1863 while ($source_file_line_number < scalar(@source_file_lines)) {
1864 print TARGET_FILE $source_file_lines[$source_file_line_number];
1865 $source_file_line_number++;
1866 }
1867
1868 close(TARGET_FILE);
1869}
1870
1871
1872# ==========================================================================================
1873# GREENSTONE3 FUNCTIONS
1874
1875sub get_all_chunks_for_gs3
1876{
1877 # The code of the target language (ensure it is lowercase)
1878 my $target_language_code = lc(shift(@_));
1879 my $translation_file_key = lc(shift(@_));
1880
1881 # Check that the necessary arguments were supplied
1882 if (!$target_language_code) {
1883 &throw_fatal_error("Missing command argument.");
1884 }
1885
1886 # Get (and check) the translation configuration
1887 # my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
1888
1889 my %source_files_key_to_text_mapping = ();
1890 my %target_files_key_to_text_mapping = ();
1891 my %source_files_key_to_last_update_date_mapping = ();
1892 my %target_files_key_to_last_update_date_mapping = ();
1893
1894 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1895
1896 &log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
1897 &log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
1898
1899 my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1900 return $xml_response;
1901}
1902
1903
1904sub get_first_n_chunks_requiring_work_for_gs3
1905{
1906 # The code of the target language (ensure it is lowercase)
1907 my $target_language_code = lc(shift(@_));
1908 # The key of the file to translate (ensure it is lowercase)
1909 my $translation_file_key = lc(shift(@_));
1910 # The number of chunks to return (defaults to one if not specified)
1911 my $num_chunks_to_return = shift(@_) || "1";
1912
1913 # Check that the necessary arguments were supplied
1914 if (!$target_language_code || !$translation_file_key) {
1915 &throw_fatal_error("Missing command argument.");
1916 }
1917
1918 my %source_files_key_to_text_mapping = ();
1919 my %target_files_key_to_text_mapping = ();
1920 my %source_files_key_to_last_update_date_mapping = ();
1921 my %target_files_key_to_last_update_date_mapping = ();
1922
1923 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1924 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1925
1926 # Determine the target file chunks requiring translation
1927 my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
1928 # Determine the target file chunks requiring updating
1929 my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1930 &log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
1931 &log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
1932
1933 my $download_target_filepath = "";
1934
1935
1936 # ****** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION ****** #
1937
1938 # if there is no copy of the language files for download, there's also no link to the spreadsheet
1939 # for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
1940 # interface *.properties files,and link to that zip as the file for offline translation.
1941 # Selecting only properties files for English and the language they're working on (if the last exists)
1942
1943 # tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
1944 # will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
1945
1946 my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
1947 my $tar_cmd = "tar -cvzf $zip";
1948
1949
1950 # store cur dir and cd to gsdlhome to generate the correct path in the zip file
1951 my $curdir = `pwd`;
1952 chdir $gsdl_root_directory;
1953
1954 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "greenstone3", \@gs3_interface_files);
1955 $tar_cmd .= " " . &get_gs3_zip_file_listing($target_language_code, "gs3-collection-configs", \@gs3_col_cfg_files);
1956
1957 # tar command will overwrite the previous version, but want to check we've created it
1958 if(&FileUtils::fileExists($zip)) {
1959 &FileUtils::removeFiles($zip);
1960 }
1961
1962 #my $tar_result = system($tar_cmd); # works but then interface breaks
1963 `$tar_cmd`;
1964 my $tar_result = $?;
1965
1966 if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
1967 $download_target_filepath = $zip;
1968 } else {
1969 &log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
1970 }
1971
1972 # change back to original working directory (cgi-bin/linux probably)
1973 chdir $curdir;
1974
1975 # ************** END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 ************* #
1976
1977
1978 my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
1979 \@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
1980 $num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
1981 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
1982
1983 return $xml_response;
1984}
1985
1986# helper function
1987# gets the listing of gs3 files for a gs3 interface module (gs3interface, gs3colcfg)
1988# formatted correctly to go into a zip file
1989sub get_gs3_zip_file_listing
1990{
1991 my $target_language_code = shift(@_);
1992 my $sourcedir = shift(@_);
1993 my $files_array = shift(@_); # reference to an array of the interfaces files for the gs3 module
1994
1995 my $filelisting = "";
1996 foreach my $interface_file (@$files_array) {
1997
1998 my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
1999 my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
2000
2001 $filelisting = "$filelisting $source_filepath";
2002 if(&FileUtils::fileExists($target_filepath)) {
2003 $filelisting = "$filelisting $target_filepath";
2004 }
2005 }
2006
2007 return $filelisting;
2008}
2009
2010sub get_uptodate_chunks_for_gs3
2011{
2012 # The code of the target language (ensure it is lowercase)
2013 my $target_language_code = lc(shift(@_));
2014 # The key of the file to translate (ensure it is lowercase)
2015 my $translation_file_key = lc(shift(@_));
2016 # The number of chunks to return (defaults to one if not specified)
2017 my $num_chunks_to_return = shift(@_) || "1";
2018
2019 # Check that the necessary arguments were supplied
2020 if (!$target_language_code || !$translation_file_key) {
2021 &throw_fatal_error("Missing command argument.");
2022 }
2023
2024 my %source_files_key_to_text_mapping = ();
2025 my %target_files_key_to_text_mapping = ();
2026 my %source_files_key_to_last_update_date_mapping = ();
2027 my %target_files_key_to_last_update_date_mapping = ();
2028
2029 &build_gs3_configuration($translation_file_key, $target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
2030 \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2031
2032
2033 # Chunks needing updating are those in the target file that have been more recently edited in the source file
2034 # All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
2035 my @uptodate_target_file_keys = ();
2036 foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
2037 my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
2038 my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
2039
2040 # print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
2041
2042 if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
2043 # &log_message("Chunk with key $chunk_key needs updating.");
2044 push(@uptodate_target_file_keys, $chunk_key);
2045 }
2046 }
2047
2048 my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
2049
2050 return $xml_response;
2051}
2052
2053
2054sub build_gs3_configuration
2055{
2056 my ($translation_file_key, $target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
2057 $source_files_key_to_gti_comment_or_last_updated_mapping, $target_files_key_to_gti_comment_or_last_updated_mapping, $get_gti_comments_not_last_updated) = @_;
2058
2059 my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
2060 my $files_array = \@gs3_interface_files;
2061
2062 if($translation_file_key eq "gs3colcfg") {
2063 $source_file_directory = "gs3-collection-configs";
2064 $files_array = \@gs3_col_cfg_files;
2065 }
2066 my $translation_file_type = "resource_bundle";
2067
2068 foreach my $interface_file_key (@$files_array) {
2069
2070 &log_message("Greenstone 3 interface file: " . $interface_file_key);
2071
2072 # Parse the source language and target language files
2073 my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
2074 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2075 my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
2076 my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2077 #my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2078
2079 my %source_file_key_to_gti_comment_or_last_updated_mapping;
2080 if($get_gti_comments_not_last_updated) {
2081 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2082 } else {
2083 %source_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
2084 }
2085
2086 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
2087 my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
2088 my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
2089 my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2090 #my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2091
2092 my %target_file_key_to_gti_comment_or_last_updated_mapping;
2093 if($get_gti_comments_not_last_updated) {
2094 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2095 } else {
2096 %target_file_key_to_gti_comment_or_last_updated_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
2097 }
2098
2099
2100 # Filter out any automatically translated chunks
2101 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2102 if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
2103 delete $source_file_key_to_line_mapping{$chunk_key};
2104 delete $target_file_key_to_line_mapping{$chunk_key};
2105 }
2106 }
2107
2108 &log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
2109 &log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
2110
2111 foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
2112 my $global_chunk_key = "$interface_file_key.$chunk_key";
2113 $source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
2114 $source_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2115
2116 if (defined $target_file_key_to_text_mapping{$chunk_key}) {
2117 $target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
2118 $target_files_key_to_gti_comment_or_last_updated_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_or_last_updated_mapping{$chunk_key};
2119 }
2120 }
2121 }
2122}
2123
2124
2125sub write_translated_gs3interface
2126{
2127 my $translation_file_key = shift(@_);
2128 my $source_file_key_to_text_mapping = shift(@_);
2129 my $target_file_key_to_text_mapping = shift(@_);
2130 my $target_file_key_to_gti_comment_mapping = shift(@_);
2131 my $target_language_code = shift(@_);
2132
2133 my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
2134
2135 my %translated_interface_file_keys = ();
2136 foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
2137 $chunk_key =~ /^([^\.]+)?\.(.*)$/;
2138 if (!defined $translated_interface_file_keys{$1}) {
2139 &log_message("Updated interface file: " . $1);
2140 $translated_interface_file_keys{$1}="";
2141 }
2142 }
2143 &log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
2144
2145 my $source_file_directory = "greenstone3";
2146 $source_file_directory = "gs3-collection-configs" if $translation_file_key eq "gs3colcfg";
2147
2148 foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
2149
2150 # Build a mapping from chunk key to source file line, and from source file line to chunk key
2151 my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
2152 my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
2153 my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
2154 my %source_file_line_to_key_mapping = ();
2155 foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
2156 $source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
2157 }
2158
2159 # Write the new target file
2160 my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
2161 my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
2162 if (!open(TARGET_FILE, ">$target_file_path")) {
2163 &throw_fatal_error("Could not write target file $target_file_path.");
2164 }
2165
2166 # Model the new target file on the source file, with the target file translations
2167 my $source_file_line_number = 0;
2168 foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
2169 # Fill in the gaps before this chunk starts
2170 my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
2171 my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
2172 while ($source_file_line_number < $source_file_chunk_starting_line_number) {
2173 print TARGET_FILE $source_file_lines[$source_file_line_number];
2174 $source_file_line_number++;
2175 }
2176 $source_file_line_number = $source_file_chunk_finishing_line_number + 1;
2177
2178 my $chunk_key = $source_file_line_to_key_mapping{$line_key};
2179 my $global_chunk_key = "$interface_file_key.$chunk_key";
2180 my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
2181 my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} || "";
2182
2183 # If no translation exists for this chunk, show this, and move on
2184 if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
2185 print TARGET_FILE "# -- Missing translation: $chunk_key\n";
2186 next;
2187 }
2188
2189 print TARGET_FILE "$chunk_key:$target_file_chunk_text";
2190 if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
2191 print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
2192 }
2193 print TARGET_FILE "\n";
2194 }
2195
2196 close(TARGET_FILE);
2197 }
2198}
2199
2200&main(@ARGV);
Note: See TracBrowser for help on using the repository browser.