Context Navigation

source: main/trunk/greenstone2/bin/script/gti.pl@ 29411

Last change on this file since 29411 was 29411, checked in by ak19, 9 years ago
There is rudimentary support for translating Greenstone 3 online via the GTI, activating that support, with slight modifications.
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 87.9 KB

Rev	Line
[10019]	1	#!/usr/bin/perl -w
	2
	3	###########################################################################
	4	#
	5	# gti.pl
	6	#
	7	# A component of the Greenstone digital library software
	8	# from the New Zealand Digital Library Project at the
	9	# University of Waikato, New Zealand.
	10	#
	11	# Copyright (C) 2005 New Zealand Digital Library Project
	12	#
	13	# This program is free software; you can redistribute it and/or modify
	14	# it under the terms of the GNU General Public License as published by
	15	# the Free Software Foundation; either version 2 of the License, or
	16	# (at your option) any later version.
	17	#
	18	# This program is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	# GNU General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU General Public License
	24	# along with this program; if not, write to the Free Software
	25	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	26	#
	27	###########################################################################
	28
	29
	30	BEGIN {
	31	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
	32	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
	33	}
	34
[24627]	35
[10019]	36	use iso639;
	37	use strict;
	38	use util;
	39
	40	my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
	41	my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
[28976]	42	my $source_language_code = "en"; # This is non-negotiable
[10019]	43
	44	my $gti_translation_files =
[24627]	45	[ # Greenstone macrofiles
	46	{ 'key' => "coredm",
[10019]	47	'file_type' => "macrofile",
	48	'source_file' => "macros/english.dm",
[11224]	49	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[24627]	50
	51	{ 'key' => "auxdm",
[10019]	52	'file_type' => "macrofile",
	53	'source_file' => "macros/english2.dm",
[11224]	54	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
[10019]	55
[29411]	56	#{ 'key' => "paperspastdm",
	57	# 'file_type' => "macrofile",
	58	# 'source_file' => "macros/paperspast-english.dm",
	59	# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[25285]	60
[24627]	61	# GLI dictionary
	62	{ 'key' => "glidict",
[13946]	63	'file_type' => "resource_bundle",
	64	'source_file' => "gli/classes/dictionary.properties",
	65	'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
[10114]	66
[24627]	67	# GLI help
	68	{ 'key' => "glihelp",
[13946]	69	'file_type' => "greenstone_xml",
	70	'source_file' => "gli/help/en/help.xml",
	71	'target_file' => "gli/help/{target_language_code}/help.xml" },
[11602]	72
[24627]	73	# Greenstone Perl modules
	74	{ 'key' => "perlmodules",
[10019]	75	'file_type' => "resource_bundle",
[11634]	76	'source_file' => "perllib/strings.properties",
	77	'target_file' => "perllib/strings_{target_language_code}.properties" },
[10019]	78
[28337]	79	# Greenstone Installer interface
	80	{ 'key' => "gsinstaller",
	81	'file_type' => "resource_bundle",
	82	'source_file' => "gsinstaller/LanguagePack.properties",
	83	'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
	84
[24627]	85	# Greenstone tutorial exercises
	86	# { 'key' => "tutorials",
	87	# 'file_type' => "greenstone_xml",
	88	# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
	89	# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
[11366]	90
[24627]	91	# new Greenstone.org
	92	{ 'key' => "greenorg",
	93	'file_type' => "resource_bundle",
	94	'source_file' => "greenstoneorg/website/classes/Gsc.properties",
	95	'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
[29411]	96	},
	97
	98	{ 'key' => "gs3interface",
	99	'file_type' => "resource_bundle",
	100	'source_file' => "greenstone3",
	101	'target_file' => "greenstone3"
[24627]	102	}
	103	];
[10019]	104
[29411]	105	my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_default2", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
[11026]	106
[26547]	107	# Auxilliary GS3 interface files. This list is not used at present
	108	# Combine with above list if generating translation spreadsheet for all interface files
	109	my @gs3_aux_interface_files = ("GATEServices", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2", "QBRWebServicesHelp", "Visualizer");
[18460]	110
[26547]	111	# Not: i18n, log4j
	112
[10019]	113	sub main
	114	{
	115	# Get the command to process, and any arguments
	116	my $gti_command = shift(@_);
	117	my @gti_command_arguments = @_;
[18460]	118	my $module = $_[1];
[26544]	119
	120	# for GS3, set gsdl_root_dir to GSDL3HOME
[29411]	121	#if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
	122	#if($ENV{'GSDL3SRCHOME'}) {
	123	# $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
	124	# $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
	125	#}
	126	#}
[24627]	127
[10019]	128	# Open the GTI log file for appending, or write to STDERR if that fails
	129	if (!open(GTI_LOG, ">>$gti_log_file")) {
[24627]	130	open(GTI_LOG, ">&STDERR");
[10019]	131	}
[24627]	132
[10019]	133	# Log the command that launched this script
	134	&log_message("Command: $0 @ARGV");
[24627]	135
[10019]	136	# Check that a command was supplied
	137	if (!$gti_command) {
[24627]	138	&throw_fatal_error("Missing command.");
[18460]	139	}
[24627]	140
[10019]	141	# Process the command
[13948]	142	if ($gti_command =~ /^get-all-chunks$/i) {
[24627]	143	# Check that GS3 interface is the target
[26544]	144	if ($module eq "gs3interface") {
[24627]	145	print &get_all_chunks_for_gs3(@gti_command_arguments);
	146	} else {
	147	print &get_all_chunks(@gti_command_arguments);
	148	}
[13948]	149	}
[24627]	150	elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
	151	if ($module eq "gs3interface") {
	152	print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
	153	} else {
	154	print &get_first_n_chunks_requiring_work(@gti_command_arguments);
	155	}
[10019]	156	}
[25249]	157	elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
	158	if ($module eq "gs3interface") {
	159	print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
	160	} else {
	161	print &get_uptodate_chunks(@gti_command_arguments);
	162	}
	163	}
[24627]	164	elsif ($gti_command =~ /^get-language-status$/i) {
	165	print &get_language_status(@gti_command_arguments);
[10019]	166	}
[24627]	167	elsif ($gti_command =~ /^search-chunks$/i) {
	168	print &search_chunks(@gti_command_arguments);
[10019]	169	}
[24627]	170	elsif ($gti_command =~ /^submit-translations$/i) {
	171	# This command cannot produce any output since it reads input
	172	&submit_translations(@gti_command_arguments);
[10019]	173	}
[24627]	174	elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
	175	# This command cannot produce any output since it reads input
	176	&create_glihelp_zip_file(@gti_command_arguments);
[18460]	177	}
[24627]	178	else {
	179	# The command was not recognized
	180	&throw_fatal_error("Unknown command \"$gti_command\".");
	181	}
[10019]	182	}
	183
	184
	185	sub throw_fatal_error
	186	{
	187	my $error_message = shift(@_);
[24627]	188
[10019]	189	# Write an XML error response
	190	print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	191	print "<GTIResponse>\n";
	192	print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
	193	print "</GTIResponse>\n";
[24627]	194
[10019]	195	# Log the error message, then die
	196	&log_message("Error: $error_message");
	197	die "\n";
	198	}
	199
	200
	201	sub log_message
	202	{
	203	my $log_message = shift(@_);
	204	print GTI_LOG time() . " -- " . $log_message . "\n";
	205	}
	206
	207
[13948]	208	sub get_all_chunks
	209	{
	210	# The code of the target language (ensure it is lowercase)
	211	my $target_language_code = lc(shift(@_));
	212	# The key of the file to translate (ensure it is lowercase)
	213	my $translation_file_key = lc(shift(@_));
[24627]	214
[13948]	215	# Check that the necessary arguments were supplied
	216	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	217	&throw_fatal_error("Missing command argument.");
[13948]	218	}
[24627]	219
[13948]	220	# Get (and check) the translation configuration
	221	my ($source_file, $target_file, $translation_file_type)
	222	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	223
[13948]	224	# Parse the source language and target language files
	225	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	226	my @source_file_lines = &read_file_lines($source_file_path);
	227	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	228
	229	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	230	my @target_file_lines = &read_file_lines($target_file_path);
	231	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	232
[13948]	233	# Filter out any automatically translated chunks
	234	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	235	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	236	delete $source_file_key_to_line_mapping{$chunk_key};
	237	delete $target_file_key_to_line_mapping{$chunk_key};
	238	}
[13948]	239	}
[24627]	240
[13948]	241	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	242	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	243	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	244	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	245
[13948]	246	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	247	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[24627]	248
[18460]	249	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
[13948]	250
	251	return $xml_response;
	252	}
	253
	254
[25249]	255	sub get_uptodate_chunks
	256	{
	257	# The code of the target language (ensure it is lowercase)
	258	my $target_language_code = lc(shift(@_));
	259	# The key of the file to translate (ensure it is lowercase)
	260	my $translation_file_key = lc(shift(@_));
	261
	262	# Check that the necessary arguments were supplied
	263	if (!$target_language_code \|\| !$translation_file_key) {
	264	&throw_fatal_error("Missing command argument.");
	265	}
	266
	267	# Get (and check) the translation configuration
	268	my ($source_file, $target_file, $translation_file_type)
	269	= &get_translation_configuration($target_language_code, $translation_file_key);
	270
	271	# Parse the source language and target language files
	272	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	273	my @source_file_lines = &read_file_lines($source_file_path);
	274	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	275
	276	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	277	my @target_file_lines = &read_file_lines($target_file_path);
	278	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	279
	280	# Filter out any automatically translated chunks
	281	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	282	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	283	delete $source_file_key_to_line_mapping{$chunk_key};
	284	delete $target_file_key_to_line_mapping{$chunk_key};
	285	}
	286	}
	287
	288	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	289	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	290	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	291	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	292
	293	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	294	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	295
	296
	297	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	298	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	299	my @uptodate_target_file_keys = ();
	300	foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
	301	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
	302	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
	303
	304	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	305
	306	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	307	# &log_message("Chunk with key $chunk_key needs updating.");
	308	push(@uptodate_target_file_keys, $chunk_key);
	309	}
	310	}
	311
	312	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	313
	314	return $xml_response;
	315	}
	316
	317
[10019]	318	sub get_first_n_chunks_requiring_work
	319	{
	320	# The code of the target language (ensure it is lowercase)
	321	my $target_language_code = lc(shift(@_));
	322	# The key of the file to translate (ensure it is lowercase)
	323	my $translation_file_key = lc(shift(@_));
	324	# The number of chunks to return (defaults to one if not specified)
	325	my $num_chunks_to_return = shift(@_) \|\| "1";
[24627]	326
[10019]	327	# Check that the necessary arguments were supplied
	328	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	329	&throw_fatal_error("Missing command argument.");
[10019]	330	}
[24627]	331
[10019]	332	# Get (and check) the translation configuration
	333	my ($source_file, $target_file, $translation_file_type)
	334	= &get_translation_configuration($target_language_code, $translation_file_key);
[28755]	335
[10019]	336	# Parse the source language and target language files
	337	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	338	my @source_file_lines = &read_file_lines($source_file_path);
	339	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
[14258]	340
[10019]	341	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	342	my @target_file_lines = &read_file_lines($target_file_path);
	343	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	344
[10019]	345	# Filter out any automatically translated chunks
	346	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	347	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	348	delete $source_file_key_to_line_mapping{$chunk_key};
	349	delete $target_file_key_to_line_mapping{$chunk_key};
	350	}
[10019]	351	}
[24627]	352
[10019]	353	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	354	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	355	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	356	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	357
[10019]	358	# Determine the target file chunks requiring translation
	359	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	360	&log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
[24627]	361
[10019]	362	# Determine the target file chunks requiring updating
	363	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	364	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	365	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	366	&log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
[24627]	367
[18460]	368	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	369
[10040]	370	return $xml_response;
[10019]	371	}
	372
	373
	374	sub get_language_status
	375	{
	376	# The code of the target language (ensure it is lowercase)
	377	my $target_language_code = lc(shift(@_));
[24627]	378
[10019]	379	# Check that the necessary arguments were supplied
	380	if (!$target_language_code) {
[24627]	381	&throw_fatal_error("Missing command argument.");
[10019]	382	}
[24627]	383
[10040]	384	# Form an XML response to the command
	385	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	386	$xml_response .= "<GTIResponse>\n";
	387	$xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
[24627]	388
[18460]	389	foreach my $translation_file (@$gti_translation_files) {
[24627]	390	my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
	391	my $target_file_name = "";
	392
	393	if ($translation_file->{'key'} eq "gs3interface") {
	394	my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
	395	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
	396
	397	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	398	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	399
	400	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	401	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	402	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	403	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
[18460]	404	}
[24627]	405	else {
	406	# Get (and check) the translation configuration
	407	my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
	408	$target_file_name = $target_file;
	409
	410	# Parse the source language and target language files
	411	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	412	my @source_file_lines = &read_file_lines($source_file_path);
	413	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	414
	415	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	416	my @target_file_lines = &read_file_lines($target_file_path);
	417	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	418
	419	# Filter out any automatically translated chunks
	420	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	421	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	422	delete $source_file_key_to_line_mapping{$chunk_key};
	423	delete $target_file_key_to_line_mapping{$chunk_key};
	424	}
	425	}
	426
	427	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	428	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	429
	430	# Determine the target file chunks requiring translation
	431	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	432
	433	# Determine the target file chunks requiring updating
	434	my @target_file_keys_requiring_updating = ();
	435	if (-e $target_file_path) {
	436	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	437	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	438	@target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	439	}
	440
	441	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	442	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	443	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	444	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
	445	}
	446
	447	&log_message("Status of " . $translation_file->{'key'});
	448	&log_message("Number of source chunks: " . $num_source_chunks);
	449	&log_message("Number of target chunks: " . $num_target_chunks);
	450	&log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
	451	&log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
	452
	453	$xml_response .= " <TranslationFile"
[10019]	454	. " key=\"" . $translation_file->{'key'} . "\""
[18460]	455	. " target_file_path=\"" . $target_file_name . "\""
	456	. " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
	457	. " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
	458	. " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
[10019]	459	}
[24627]	460
[10040]	461	$xml_response .= " </LanguageStatus>\n";
[24627]	462
[10040]	463	$xml_response .= "</GTIResponse>\n";
	464	return $xml_response;
[10019]	465	}
	466
	467
	468	sub search_chunks
	469	{
	470	# The code of the target language (ensure it is lowercase)
	471	my $target_language_code = lc(shift(@_));
	472	# The key of the file to translate (ensure it is lowercase)
	473	my $translation_file_key = lc(shift(@_));
	474	# The query string
[10020]	475	my $query_string = join(' ', @_);
[24627]	476
[10019]	477	# Check that the necessary arguments were supplied
	478	if (!$target_language_code \|\| !$translation_file_key \|\| !$query_string) {
[24627]	479	&throw_fatal_error("Missing command argument.");
[10019]	480	}
[24627]	481
[18460]	482	my ($source_file, $target_file, $translation_file_type) = ();
	483	my %source_file_key_to_text_mapping = ();
	484	my %target_file_key_to_text_mapping = ();
	485
	486
	487	if ($translation_file_key ne "gs3interface") {
[24627]	488	# Get (and check) the translation configuration
	489	($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	490
	491	# Parse the source language and target language files
	492	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	493	my @source_file_lines = &read_file_lines($source_file_path);
	494	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	495
	496	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	497	my @target_file_lines = &read_file_lines($target_file_path);
	498	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	499
	500	# Filter out any automatically translated chunks
	501	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	502	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	503	delete $source_file_key_to_line_mapping{$chunk_key};
	504	delete $target_file_key_to_line_mapping{$chunk_key};
	505	}
	506	}
	507
	508	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	509	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[10019]	510	}
[18460]	511	else {
[24627]	512	# Not needed in this case
	513	my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
	514	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	515	\%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
[18460]	516	}
[24627]	517
[10019]	518	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	519	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	520
[10019]	521	# Determine the target file chunks matching the query
	522	my @target_file_keys_matching_query = ();
	523	foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
[24627]	524	my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
	525	if ($target_file_text =~ /$query_string/i) {
	526	# &log_message("Chunk with key $chunk_key matches query.");
	527	push(@target_file_keys_matching_query, $chunk_key);
	528	}
[10019]	529	}
[24627]	530
[10040]	531	# Form an XML response to the command
	532	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	533	$xml_response .= "<GTIResponse>\n";
[24627]	534
[10040]	535	$xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
[10019]	536	foreach my $chunk_key (@target_file_keys_matching_query) {
[24627]	537	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
	538
	539	$xml_response .= " <Chunk key=\"$chunk_key\">\n";
	540	$xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
	541	$xml_response .= " </Chunk>\n";
[10019]	542	}
[10040]	543	$xml_response .= " </ChunksMatchingQuery>\n";
[24627]	544
[10040]	545	$xml_response .= "</GTIResponse>\n";
	546	return $xml_response;
[10019]	547	}
	548
	549
[10050]	550	sub submit_translations
[10019]	551	{
	552	# The code of the target language (ensure it is lowercase)
	553	my $target_language_code = lc(shift(@_));
	554	# The key of the file to translate (ensure it is lowercase)
	555	my $translation_file_key = lc(shift(@_));
[12484]	556	# The username of the translation submitter
	557	my $submitter_username = shift(@_);
[10027]	558	# Whether to submit a target chunk even if it hasn't changed
	559	my $force_submission_flag = shift(@_);
[24627]	560
[10019]	561	# Check that the necessary arguments were supplied
[12484]	562	if (!$target_language_code \|\| !$translation_file_key \|\| !$submitter_username) {
[24627]	563	&log_message("Fatal error (but cannot be thrown): Missing command argument.");
	564	die "\n";
[10019]	565	}
[18460]	566
	567	my %source_file_key_to_text_mapping = ();
	568	my %source_file_key_to_gti_comment_mapping = ();
	569	my %target_file_key_to_text_mapping = ();
	570	my %target_file_key_to_gti_comment_mapping = ();
[24627]	571
[18460]	572	my (@source_file_lines, @target_file_lines) = ();
	573	my ($source_file, $target_file, $translation_file_type);
[24627]	574
[18460]	575
	576	if ($translation_file_key ne "gs3interface") {
[24627]	577	# Get (and check) the translation configuration
	578	($source_file, $target_file, $translation_file_type)
[18460]	579	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	580
	581	# Parse the source language and target language files
	582	@source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	583	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	584	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	585	%source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	586
	587	@target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	588	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	589	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	590	%target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[18460]	591	}
	592	else {
[24627]	593	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	594	\%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
[18460]	595	}
[10019]	596	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	597	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	598
[10019]	599	# Submission date
	600	my $day = (localtime)[3];
	601	my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
	602	my $year = (localtime)[5] + 1900;
	603	my $submission_date = "$day-$month-$year";
[24627]	604
[10019]	605	open(SUBMISSION, "-");
	606	my @submission_lines = <SUBMISSION>;
	607	close(SUBMISSION);
[24627]	608
[10019]	609	# Remove any nasty carriage returns
[18460]	610	# &log_message("Submission:");
[10019]	611	foreach my $submission_line (@submission_lines) {
[24627]	612	$submission_line =~ s/\r$//;
	613	#&log_message(" $submission_line");
[10019]	614	}
[24627]	615
[10019]	616	my %source_file_key_to_submission_mapping = ();
	617	my %target_file_key_to_submission_mapping = ();
	618	for (my $i = 0; $i < scalar(@submission_lines); $i++) {
[24627]	619	# Read source file part of submission
	620	if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
	621	my $chunk_key = $1;
	622
	623	# Read the source file text
	624	my $source_file_chunk_text = "";
	625	$i++;
	626	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
	627	$source_file_chunk_text .= $submission_lines[$i];
	628	$i++;
	629	}
	630	$source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	631	$source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
	632
	633	#&log_message("Source file key: $chunk_key");
	634	#&log_message("Source file text: $source_file_chunk_text");
	635	$source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
	636	}
	637
	638	# Read target file part of submission
	639	if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
	640	my $chunk_key = $1;
	641
	642	# Read the target file text
	643	my $target_file_chunk_text = "";
	644	$i++;
	645	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
	646	$target_file_chunk_text .= $submission_lines[$i];
	647	$i++;
	648	}
	649	$target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	650	$target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
	651
	652	#&log_message("Target file key: $chunk_key");
	653	#&log_message("Target file text: $target_file_chunk_text");
	654	$target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
	655	}
[10019]	656	}
[24627]	657
[10019]	658	# -----------------------------------------
	659	# Validate the translation submissions
	660	# -----------------------------------------
[24627]	661
[10019]	662	# Check that the translations are valid
	663	foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
[24627]	664	# Make sure the submitted chunk still exists in the source file
	665	if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
	666	&log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
	667	delete $source_file_key_to_submission_mapping{$chunk_key};
	668	delete $target_file_key_to_submission_mapping{$chunk_key};
	669	next;
	670	}
	671
	672	# Make sure the submitted source chunk matches the source file chunk
[28518]	673	if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
	674	#if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
[28503]	675	#print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
	676	#print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
	677
[24627]	678	&log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
	679	&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
	680	&log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
	681	delete $source_file_key_to_submission_mapping{$chunk_key};
	682	delete $target_file_key_to_submission_mapping{$chunk_key};
	683	next;
	684	}
[10027]	685	}
[24627]	686
[10027]	687	# Apply the submitted translations
	688	foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
[24627]	689	# Only apply the submission if it is a change, unless -force_submission has been specified
	690	if ($force_submission_flag \|\| !defined($target_file_key_to_text_mapping{$chunk_key}) \|\| $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
	691	$target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
	692	$target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
	693	}
[10019]	694	}
[18460]	695
	696	if ($translation_file_key ne "gs3interface") {
[24627]	697	eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	698	} else {
[24627]	699	eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	700	}
	701	}
[10019]	702
[18460]	703
	704	sub create_glihelp_zip_file
	705	{
	706	my $target_language_code = shift(@_);
	707	my $translation_file_key = "glihelp";
	708
	709	&log_message("Creating GLI Help zip file for $target_language_code");
[24627]	710
[18460]	711	my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
	712
	713	my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
[28977]	714	my $oldclasspath = $classpath;
[18460]	715	if ( ! -e $classpath) {
[28977]	716	$classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
	717	}
	718	if ( ! -e $classpath) {
	719	&throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
	720	}
	721
[28991]	722
	723	my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
	724	my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
	725	my $os = $^O;
	726	my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
	727	my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
	728	$classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
	729
[18460]	730	my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
	731	$gli_help_directory = &util::filename_cat($gli_help_directory, "help");
	732
	733	my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
	734	if ( ! -e $gen_many_html_xsl_filepath) {
[24627]	735	&throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
[18460]	736	}
[24627]	737
[18460]	738	my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
	739	my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
	740
	741	my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
	742	$target_file_directory = $target_file_directory."/";
[24627]	743
[18460]	744	my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
[28991]	745
	746	# if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
	747	my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
	748	if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
	749
	750	# copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
	751	my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
[28992]	752	my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
[28991]	753	&FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
	754
	755	# In gli/help/<newlang>/help.xml, replace all occurrences of
	756	# <Text id="1">This text in en will be removed for new langcode</Text>
	757	# with <!-- Missing translation: 1 -->
	758	open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
	759	my $help_xml_contents;
	760	# Read in the entire contents of the file in one hit
	761	sysread(FIN, $help_xml_contents, -s FIN);
	762	close(FIN);
[24627]	763
[28991]	764	$help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
	765
	766	open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
	767	print FOUT $help_xml_contents;
	768	close(FOUT);
	769	}
	770
[24627]	771	my $perl_exec = &util::get_perl_exec();
	772	my $java_exec = "java";
	773	if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
	774	$java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
	775	}
[28991]	776
	777	#my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	778	my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	779	#&throw_fatal_error("RAN gti command: $cmd");
[18460]	780	my $response = `$cmd`;
[28991]	781
	782	#$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
	783	$cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
[18460]	784	$response = `$cmd`;
[28991]	785
	786	# create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
	787	my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
	788	if(!&FileUtils::directoryExists($tmpdir)) {
	789	&FileUtils::makeDirectory($tmpdir);
	790	}
[28977]	791	#my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
[28991]	792	my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
[18460]	793	$cmd = "zip -rj $zip_file_path $target_file_directory -i \.htm \.xml";
[28991]	794
[18460]	795	$response = `$cmd`;
[10019]	796	}
	797
	798
	799	sub get_translation_configuration
	800	{
	801	# Get the code of the target language
	802	my $target_language_code = shift(@_);
	803	# Get the key of the file to translate
	804	my $translation_file_key = shift(@_);
[24627]	805
[10019]	806	# Read the translation data from the gti.cfg file
	807	my ($source_file, $target_file, $translation_file_type) =
	808	&get_translation_data_for($target_language_code, $translation_file_key);
[24627]	809
[10019]	810	# Check that the file to translate is defined in the gti.cfg file
	811	if (!$source_file \|\| !$target_file \|\| !$translation_file_type) {
[24627]	812	&throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
[10019]	813	}
[24627]	814
[10019]	815	# Check that the source file exists
	816	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	817	if (!-e $source_file_path) {
[24627]	818	&throw_fatal_error("Source file $source_file_path does not exist.");
[10019]	819	}
[24627]	820
[10019]	821	# Check that the source file is up to date
[11104]	822	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[10019]	823	# unless ($translation_file_is_not_in_cvs) {
[14258]	824	#my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
[24627]	825	my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
[10019]	826	if ($source_file_cvs_status =~ /^C /) {
	827	&throw_fatal_error("Source file $source_file_path conflicts with the repository.");
	828	}
	829	if ($source_file_cvs_status =~ /^M /) {
	830	&throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
	831	}
	832	# }
[24627]	833
[10019]	834	return ($source_file, $target_file, $translation_file_type);
	835	}
	836
	837
	838	sub get_translation_data_for
	839	{
	840	my ($target_language_code, $translation_file_key) = @_;
[24627]	841
[10019]	842	foreach my $translation_file (@$gti_translation_files) {
[24627]	843	# If this isn't the correct translation file, move onto the next one
	844	next if ($translation_file_key ne $translation_file->{'key'});
	845
	846	# Resolve the target language file
	847	my $target_language_file = $translation_file->{'target_file'};
	848	if ($target_language_file =~ /(\{.+\;.+\})/) {
	849	my $unresolved_target_language_file_part = $1;
	850
	851	# Check for a special case for the target language code
	852	if ($unresolved_target_language_file_part =~ /(\{\|\;)$target_language_code:([^\;]+)(\;\|\})/) {
	853	my $resolved_target_language_file_part = $2;
	854	$target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
	855	}
	856	# Otherwise use the last part as the default value
	857	else {
	858	my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
	859	$target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
[10019]	860	}
[10041]	861	}
[24627]	862
[10041]	863	# Resolve instances of {iso_639_1_target_language_name}
	864	my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
	865	$iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
	866	$target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
[24627]	867
[10041]	868	# Resolve instances of {target_language_code}
	869	$target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
[24627]	870
[10041]	871	return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
[24627]	872	}
[10019]	873
[24627]	874	return ();
[10019]	875	}
	876
	877
	878	sub read_file_lines
	879	{
	880	my ($file_path) = @_;
[24627]	881
[10019]	882	if (!open(FILE_IN, "<$file_path")) {
[24627]	883	&log_message("Note: Could not open file $file_path.");
	884	return ();
[10019]	885	}
	886	my @file_lines = <FILE_IN>;
	887	close(FILE_IN);
[24627]	888
[10019]	889	return @file_lines;
	890	}
	891
	892
	893	sub build_key_to_line_mapping
	894	{
	895	my ($file_lines, $translation_file_type) = @_;
	896	eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
	897	}
	898
	899
	900	sub build_key_to_text_mapping
	901	{
	902	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	903
[10019]	904	my %key_to_text_mapping = ();
	905	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	906	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	907	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	908
	909	my $chunk_text = @$file_lines[$chunk_starting_line];
	910	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	911	$chunk_text .= @$file_lines[$l];
	912	}
	913
	914	# Map from chunk key to text
	915	eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
[10019]	916	}
[24627]	917
[10019]	918	return %key_to_text_mapping;
	919	}
	920
	921
	922	sub build_key_to_last_update_date_mapping
	923	{
	924	my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	925
[10019]	926	# If the files aren't in CVS then we can't tell anything about what needs updating
	927	# return () if ($translation_file_is_not_in_cvs);
[24627]	928
[10019]	929	# Build a mapping from key to CVS date
	930	# Need to be careful with this mapping because the chunk keys won't necessarily all be valid
	931	my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
[24627]	932
[10019]	933	# Build a mapping from key to comment date
[12483]	934	my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
[24627]	935
[10019]	936	# Build a mapping from key to last update date (the latter of the CVS date and comment date)
	937	my %key_to_last_update_date_mapping = ();
	938	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	939	# Use the CVS date as a starting point
	940	my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
	941	$key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
	942
	943	# If a comment date exists and it is after the CVS date, use that instead
[14258]	944	# need to convert the comment date format to SVN format
[24627]	945	my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
	946	if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
	947	my $chunk_comment_date = $1;
	948	if ((!defined($chunk_cvs_date) \|\| &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
	949	$key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
	950	}
	951	}
[10019]	952	}
[24627]	953
[10019]	954	return %key_to_last_update_date_mapping;
	955	}
	956
	957
	958	sub build_key_to_cvs_date_mapping
	959	{
	960	my ($filename, $translation_file_type) = @_;
[24627]	961
	962	# Use SVN to annotate each line of the file with the date it was last edited
[10019]	963	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[24627]	964	my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
[14258]	965
[10019]	966	my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
[24627]	967
[10019]	968	my @cvs_annotated_file_lines_date = ();
	969	foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
[24627]	970	# Extract the date from the SVN annotation at the front
	971	# svn format : 2007-07-16
[14258]	972	$cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
	973
	974	push(@cvs_annotated_file_lines_date, $1);
	975
	976	# trim extra date information in svn annotation format
	977	# 15:42:49 +1200 (Wed, 21 Jun 2006)
	978	$cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
	979	}
	980
[10019]	981	# Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
	982	my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
[24627]	983
[10019]	984	my %key_to_cvs_date_mapping = ();
	985	foreach my $chunk_key (keys(%key_to_line_mapping)) {
[24627]	986	my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
	987	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
	988
	989	# Find the date this chunk was last edited, from the CVS annotation
	990	my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
	991	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	992	if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
	993	# This part of the chunk has been updated more recently
	994	$chunk_date = $cvs_annotated_file_lines_date[$l];
	995
	996	}
	997	}
	998
	999	# Map from chunk key to CVS date
	1000	$key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
[10019]	1001	}
[24627]	1002
[10019]	1003	return %key_to_cvs_date_mapping;
	1004	}
	1005
	1006
[12483]	1007	sub build_key_to_gti_comment_mapping
[10019]	1008	{
	1009	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	1010
[12483]	1011	my %key_to_gti_comment_mapping = ();
[10019]	1012	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	1013	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	1014	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	1015
	1016	my $chunk_text = @$file_lines[$chunk_starting_line];
	1017	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	1018	$chunk_text .= @$file_lines[$l];
	1019	}
	1020
	1021	# Map from chunk key to GTI comment
	1022	my $chunk_gti_comment;
	1023	eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
	1024	$key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
[10019]	1025	}
[24627]	1026
[12483]	1027	return %key_to_gti_comment_mapping;
[10019]	1028	}
	1029
	1030
	1031	sub determine_chunks_requiring_translation
	1032	{
	1033	my $source_file_key_to_text_mapping = shift(@_);
	1034	my $target_file_key_to_text_mapping = shift(@_);
[24627]	1035
[10019]	1036	# Chunks needing translation are those in the source file with no translation in the target file
	1037	my @target_file_keys_requiring_translation = ();
	1038	foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
[24627]	1039	if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
	1040	# &log_message("Chunk with key $chunk_key needs translating.");
	1041	push(@target_file_keys_requiring_translation, $chunk_key);
	1042	}
[10019]	1043	}
[24627]	1044
[10019]	1045	return @target_file_keys_requiring_translation;
	1046	}
	1047
	1048
	1049	sub determine_chunks_requiring_updating
	1050	{
	1051	my $source_file_key_to_last_update_date_mapping = shift(@_);
	1052	my $target_file_key_to_last_update_date_mapping = shift(@_);
[24627]	1053
[10019]	1054	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1055	my @target_file_keys_requiring_updating = ();
	1056	foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
[24627]	1057	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
	1058	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
[14258]	1059
	1060	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
[24627]	1061
[14258]	1062	if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
[24627]	1063	# &log_message("Chunk with key $chunk_key needs updating.");
	1064	push(@target_file_keys_requiring_updating, $chunk_key);
	1065	}
[10019]	1066	}
[24627]	1067
[10019]	1068	return @target_file_keys_requiring_updating;
	1069	}
	1070
	1071
	1072	sub is_chunk_automatically_translated
	1073	{
	1074	my ($chunk_key, $translation_file_type) = @_;
	1075	eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
	1076	}
	1077
	1078
	1079	sub make_text_xml_safe
	1080	{
	1081	my $text = shift(@_);
[11452]	1082	$text =~ s/\&/\&amp\;/g;
	1083	$text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
	1084	$text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
[11498]	1085	$text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
	1086	$text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
[10019]	1087	$text =~ s/</\&lt\;/g;
	1088	$text =~ s/>/\&gt\;/g;
	1089	return $text;
	1090	}
	1091
	1092
[11448]	1093	sub unmake_text_xml_safe
	1094	{
	1095	my $text = shift(@_);
	1096	$text =~ s/\&lt\;/</g;
	1097	$text =~ s/\&gt\;/>/g;
	1098	$text =~ s/\&amp\;/\&/g;
	1099	return $text;
	1100	}
	1101
	1102
[10019]	1103	# Returns 1 if $date1 is after $date2, 0 otherwise
[14258]	1104	sub is_date_after_cvs
[10019]	1105	{
	1106	my ($date1, $date2) = @_;
	1107	my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
[24627]	1108	"Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
	1109
[14258]	1110	if(!defined $date1) {
	1111	return 1;
	1112	}
[24627]	1113
[10019]	1114	my @date1parts = split(/-/, $date1);
	1115	my @date2parts = split(/-/, $date2);
[24627]	1116
[10019]	1117	# Compare year - nasty because we have rolled over into a new century
	1118	my $year1 = $date1parts[2];
	1119	if ($year1 < 80) {
[14258]	1120	$year1 += 2000;
[10019]	1121	}
	1122	my $year2 = $date2parts[2];
	1123	if ($year2 < 80) {
[14258]	1124	$year2 += 2000;
[10019]	1125	}
[24627]	1126
[10019]	1127	# Compare year
	1128	if ($year1 > $year2) {
[24627]	1129	return 1;
[10019]	1130	}
	1131	elsif ($year1 == $year2) {
[24627]	1132	# Year is the same, so compare month
	1133	if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
	1134	return 1;
	1135	}
	1136	elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
	1137	# Month is the same, so compare day
	1138	if ($date1parts[0] > $date2parts[0]) {
	1139	return 1;
	1140	}
	1141	}
[10019]	1142	}
[24627]	1143
[10019]	1144	return 0;
	1145	}
	1146
[14258]	1147	sub is_date_after
	1148	{
	1149	my ($date1, $date2) = @_;
	1150
	1151	if(!defined $date1) {
[24627]	1152	return 1;
[14258]	1153	}
	1154	if(!defined $date2) {
[24627]	1155	return 0;
[14258]	1156	}
	1157
	1158	# 16-Aug-2006
	1159	if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1160	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1161	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1162	$date1=$3 . "-" . $months{$2} . "-" . $1;
	1163	# print "** converted date1: $date1\n";
[14258]	1164	}
	1165	if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1166	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1167	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1168	$date2=$3 . "-" . $months{$2} . "-" . $1;
	1169	# print "** converted date2: $date2\n";
[14258]	1170	}
	1171
	1172
	1173	# 2006-08-16
	1174	my @date1parts = split(/-/, $date1);
	1175	my @date2parts = split(/-/, $date2);
	1176
	1177	# Compare year
	1178	if ($date1parts[0] > $date2parts[0]) {
[24627]	1179	return 1;
[14258]	1180	}
	1181	elsif ($date1parts[0] == $date2parts[0]) {
[24627]	1182	# Year is the same, so compare month
	1183	if ($date1parts[1] > $date2parts[1]) {
	1184	return 1;
	1185	}
	1186	elsif ($date1parts[1] == $date2parts[1]) {
	1187	# Month is the same, so compare day
	1188	if ($date1parts[2] > $date2parts[2]) {
	1189	return 1;
	1190	}
	1191	}
[14258]	1192	}
	1193
	1194	return 0;
	1195	}
[10019]	1196
[14258]	1197
[18460]	1198	sub create_xml_response_for_chunks_requiring_work
	1199	{
	1200	my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
[24627]	1201
[18460]	1202	# Form an XML response to the command
	1203	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1204	$xml_response .= "<GTIResponse>\n";
	1205	$xml_response .= " <TranslationFile"
	1206	. " key=\"" . $translation_file_key . "\""
	1207	. " target_file_path=\"" . $target_file . "\""
	1208	. " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
	1209	. " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
	1210	. " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
[24627]	1211
[18460]	1212	# Do chunks requiring translation first
	1213	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
[24627]	1214	$xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
[18460]	1215	}
	1216	else {
[24627]	1217	$xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1218	}
[24627]	1219
[18460]	1220	my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
	1221	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1222	last if ($num_chunks_to_return == 0);
	1223
	1224	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1225	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1226
	1227	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1228	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1229	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1230	$xml_response .= " </Chunk>\n";
	1231
	1232	$num_chunks_to_return--;
	1233	}
[18460]	1234
	1235	$xml_response .= " </ChunksRequiringTranslation>\n";
[24627]	1236
[18460]	1237	# Then do chunks requiring updating
	1238	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
[24627]	1239	$xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
[18460]	1240	}
	1241	else {
[24627]	1242	$xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1243	}
[24627]	1244
[18460]	1245	# foreach my $chunk_key (@target_file_keys_requiring_updating) {
	1246	@sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
	1247	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1248	last if ($num_chunks_to_return == 0);
	1249
	1250	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1251	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1252	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1253	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1254
	1255	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1256	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1257	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1258	$xml_response .= " </Chunk>\n";
	1259
	1260	$num_chunks_to_return--;
[18460]	1261	}
[24627]	1262
[18460]	1263	$xml_response .= " </ChunksRequiringUpdating>\n";
[24627]	1264
[18460]	1265	$xml_response .= "</GTIResponse>\n";
[24627]	1266
[18460]	1267	return $xml_response;
	1268	}
	1269
[25249]	1270	sub create_xml_response_for_uptodate_chunks
	1271	{
	1272	my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
	1273
	1274	# Form an XML response to the command
	1275	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1276	$xml_response .= "<GTIResponse>\n";
	1277	$xml_response .= " <TranslationFile"
	1278	. " key=\"" . $translation_file_key . "\""
	1279	. " target_file_path=\"" . $target_file . "\""
	1280	. " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
	1281
	1282
	1283	# Then do chunks requiring updating
	1284	$xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
	1285
	1286
	1287	# foreach my $chunk_key (@uptodate_target_file_keys) {
	1288	my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
	1289	foreach my $chunk_key (@sorted_chunk_keys) {
	1290
	1291	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1292	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1293	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1294	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1295
	1296	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1297	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1298	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1299	$xml_response .= " </Chunk>\n";
[18460]	1300
[25249]	1301	}
	1302
	1303	$xml_response .= " </UptodateChunks>\n";
	1304
	1305	$xml_response .= "</GTIResponse>\n";
	1306
	1307	return $xml_response;
	1308	}
	1309
[18460]	1310	sub create_xml_response_for_all_chunks
	1311	{
	1312	my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
[24627]	1313
[18460]	1314	# Form an XML response to the command
	1315	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1316	$xml_response .= "<GTIResponse>\n";
	1317	$xml_response .= " <TranslationFile"
	1318	. " key=\"" . $translation_file_key . "\""
	1319	. " target_file_path=\"" . $target_file . "\"\/>\n";
	1320
	1321	# Do all the chunks
	1322	$xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
[24627]	1323
[18460]	1324	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
	1325	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1326	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1327	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
	1328
	1329	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1330	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1331	if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
	1332	my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1333	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
	1334	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1335	}
	1336	else {
	1337	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1338	}
	1339
	1340	$xml_response .= " </Chunk>\n";
[18460]	1341	}
	1342	$xml_response .= " </Chunks>\n";
	1343
	1344	$xml_response .= "</GTIResponse>\n";
	1345	return $xml_response;
	1346	}
	1347
	1348
	1349
[10019]	1350	# ==========================================================================================
	1351	# MACROFILE FUNCTIONS
	1352
	1353	sub build_key_to_line_mapping_for_macrofile
	1354	{
	1355	my (@file_lines) = @_;
[24627]	1356
[10019]	1357	my $macro_package;
	1358	my %chunk_key_to_line_mapping = ();
	1359	# Process the contents of the file, line by line
	1360	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1361	my $line = $file_lines[$i];
	1362	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1363
	1364	# Check if a new package is being defined
	1365	if ($line =~ m/^package\s+(.+)/) {
	1366	$macro_package = $1;
[10041]	1367	}
[24627]	1368
	1369	# Line contains a macro name
	1370	elsif ($line =~ m/^(_\w+_)/) {
	1371	my $macro_key = $1;
	1372	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1373
	1374	# While there is still text of the macro to go...
	1375	my $startline = $i;
	1376	while ($line !~ /\}$/) {
	1377	$i++;
	1378	if ($i == scalar(@file_lines)) {
	1379	&throw_fatal_error("Could not find end of macro $macro_key.");
	1380	}
	1381	$line = $file_lines[$i];
	1382	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1383	}
	1384
[10019]	1385	# The chunk key consists of the package name and the macro key
	1386	my $chunk_key = $macro_package . "." . $macro_key;
	1387	# Map from chunk key to line
	1388	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1389	}
[24627]	1390
[10019]	1391	# Icon: line in format ## "image text" ## image_type ## macro_name ##
	1392	elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
[24627]	1393	# The chunk key consists of package name and macro key
	1394	my $chunk_key = $macro_package . "." . $1;
	1395	# Map from chunk key to line
	1396	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1397	}
	1398	}
[10019]	1399
[24627]	1400	return %chunk_key_to_line_mapping;
[10019]	1401	}
	1402
	1403
	1404	sub import_chunk_from_macrofile
	1405	{
	1406	my ($chunk_text) = @_;
[24627]	1407
[10019]	1408	# Is this an icon macro??
	1409	if ($chunk_text =~ /^\#\# (.*)/) {
[24627]	1410	# Extract image macro text
	1411	$chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
	1412	$chunk_text = $1;
	1413
[10019]	1414	# Remove enclosing quotes
	1415	$chunk_text =~ s/^\"//;
	1416	$chunk_text =~ s/\"$//;
[24627]	1417	}
[10019]	1418
[24627]	1419	# No, so it must be a text macro
	1420	else {
[10019]	1421	# Remove macro key
	1422	$chunk_text =~ s/^_([^_]+)_(\s*)//;
[24627]	1423
[10019]	1424	# Remove language specifier
	1425	$chunk_text =~ s/^\[l=.\](\s)//;
[24627]	1426
[10019]	1427	# Remove braces enclosing text
	1428	$chunk_text =~ s/^{(\s)((.\|\n))}(\s)(\#.+\s)?/$2/;
[24627]	1429	}
[10019]	1430
[24627]	1431	return $chunk_text;
[10019]	1432	}
	1433
	1434
[12483]	1435	sub get_macrofile_chunk_gti_comment
[10019]	1436	{
	1437	my ($chunk_text) = @_;
[24627]	1438
[10019]	1439	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1440	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1441	return $1;
	1442	}
[10019]	1443
[24627]	1444	return undef;
[10019]	1445	}
	1446
	1447
	1448	sub is_macrofile_chunk_automatically_translated
	1449	{
	1450	my ($chunk_key) = @_;
[24627]	1451
[10019]	1452	# The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
	1453	if ($chunk_key =~ /\._(httpicon\|width\|height)/) {
[24627]	1454	return 1;
[10019]	1455	}
[24627]	1456
[10019]	1457	return 0;
	1458	}
	1459
	1460
	1461	# Use the source file to generate a target file that is formatted the same
	1462	sub write_translated_macrofile
	1463	{
	1464	my $source_file = shift(@_); # Not used
[11321]	1465	my @source_file_lines = @{shift(@_)};
[10019]	1466	my $source_file_key_to_text_mapping = shift(@_);
	1467	my $target_file = shift(@_);
[11321]	1468	my @target_file_lines = @{shift(@_)};
[10019]	1469	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1470	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1471	my $target_language_code = shift(@_);
[24627]	1472
[10126]	1473	# Build a mapping from source file line to chunk key
[11321]	1474	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
[10019]	1475	my %source_file_line_to_key_mapping = ();
	1476	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1477	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1478	}
[10126]	1479	my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
	1480	my $source_file_line_number = 0;
[24627]	1481
[10126]	1482	# Build a mapping from target file line to chunk key
[11321]	1483	my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
[10126]	1484	my %target_file_line_to_key_mapping = ();
	1485	foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
[24627]	1486	$target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10126]	1487	}
	1488	my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
[24627]	1489
[10019]	1490	# Write the new target file
	1491	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1492	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1493	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1494	}
[24627]	1495
[10126]	1496	# Use the header from the target file, to keep language and author information
	1497	if (scalar(@target_file_line_keys) > 0) {
[24627]	1498	my $target_file_line_number = 0;
	1499	my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
	1500	while ($target_file_line_number < $target_file_chunk_starting_line_number) {
	1501	my $target_file_line = $target_file_lines[$target_file_line_number];
	1502	last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
	1503	print TARGET_FILE $target_file_line;
	1504	$target_file_line_number++;
	1505	}
	1506
	1507	$source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
[10126]	1508	}
[24627]	1509
[10019]	1510	# Model the new target file on the source file, with the target file translations
[10126]	1511	foreach my $line_key (@source_file_line_keys) {
[24627]	1512	# Fill in the gaps before this chunk starts
	1513	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1514	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1515	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1516	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1517	$source_file_line_number++;
	1518	}
	1519	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1520
	1521	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1522	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1523	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1524
	1525	my $macrofile_key = $chunk_key;
	1526	$macrofile_key =~ s/^(.+?)\.//;
	1527
	1528	# If no translation exists for this chunk, show this, and move on
	1529	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1530	print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
	1531	next;
	1532	}
	1533
	1534	# Grab the source chunk text
	1535	my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
	1536	for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
	1537	$source_file_chunk .= $source_file_lines[$l];
	1538	}
	1539
	1540	# Is this an icon macro??
	1541	if ($source_file_chunk =~ /^\#\# (.*)/) {
	1542	# Escape any newline and question mark characters so the source text is replaced correctly
	1543	$source_file_chunk_text =~ s/\\/\\\\/g;
[10019]	1544	$source_file_chunk_text =~ s/\?/\\\?/g;
[24627]	1545
[10019]	1546	# Build the new target chunk from the source chunk
	1547	my $target_file_chunk = $source_file_chunk;
	1548	$target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
[10050]	1549	$target_file_chunk =~ s/(\s)*$//;
[10019]	1550	print TARGET_FILE "$target_file_chunk";
	1551	}
[24627]	1552
[10019]	1553	# No, it is just a normal text macro
	1554	else {
	1555	print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
	1556	}
[24627]	1557
[12483]	1558	# Add the "updated" comment, if one exists
	1559	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1560	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
[10050]	1561	}
	1562	print TARGET_FILE "\n";
[24627]	1563	}
[10019]	1564
[24627]	1565	close(TARGET_FILE);
[10019]	1566	}
	1567
	1568
	1569	sub sort_by_line
	1570	{
	1571	return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
	1572	}
	1573
	1574
	1575	# ==========================================================================================
	1576	# RESOURCE BUNDLE FUNCTIONS
	1577
	1578	sub build_key_to_line_mapping_for_resource_bundle
	1579	{
	1580	my (@file_lines) = @_;
[24627]	1581
[11321]	1582	my %chunk_key_to_line_mapping = ();
[10019]	1583	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1584	my $line = $file_lines[$i];
	1585	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1586
	1587	# Line contains a dictionary string
	1588	if ($line =~ /^(\S+?)[:\|=](.*)$/) {
	1589	my $chunk_key = $1;
	1590
	1591	# Map from chunk key to line
	1592	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1593	}
[10019]	1594	}
[24627]	1595
[11321]	1596	return %chunk_key_to_line_mapping;
[10019]	1597	}
	1598
	1599
	1600	sub import_chunk_from_resource_bundle
	1601	{
	1602	my ($chunk_text) = @_;
[24627]	1603
[10019]	1604	# Simple: just remove string key
[18460]	1605	$chunk_text =~ s/^(\S+?)[:\|=](\s*)//;
[10019]	1606	$chunk_text =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
[12484]	1607	$chunk_text =~ s/(\s)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.)\s*$//i;
[24627]	1608
[10019]	1609	return $chunk_text;
	1610	}
	1611
	1612
[12483]	1613	sub get_resource_bundle_chunk_gti_comment
[10019]	1614	{
	1615	my ($chunk_text) = @_;
[24627]	1616
[10019]	1617	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1618	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1619	return $1;
	1620	}
[10019]	1621
[24627]	1622	return undef;
[10019]	1623	}
	1624
	1625
	1626	sub is_resource_bundle_chunk_automatically_translated
	1627	{
	1628	# No resource bundle chunks are automatically translated
	1629	return 0;
	1630	}
	1631
	1632
	1633	sub write_translated_resource_bundle
	1634	{
	1635	my $source_file = shift(@_); # Not used
[11321]	1636	my @source_file_lines = @{shift(@_)};
[10019]	1637	my $source_file_key_to_text_mapping = shift(@_);
	1638	my $target_file = shift(@_);
[11321]	1639	my @target_file_lines = @{shift(@_)}; # Not used
[10019]	1640	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1641	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1642	my $target_language_code = shift(@_); # Not used
[24627]	1643
[10019]	1644	# Build a mapping from chunk key to source file line, and from source file line to chunk key
[11321]	1645	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
[10019]	1646	my %source_file_line_to_key_mapping = ();
	1647	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1648	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1649	}
[24627]	1650
[10019]	1651	# Write the new target file
	1652	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1653	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1654	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1655	}
[24627]	1656
[10019]	1657	# Model the new target file on the source file, with the target file translations
[10091]	1658	my $source_file_line_number = 0;
[10019]	1659	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1660	# Fill in the gaps before this chunk starts
	1661	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1662	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1663	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1664	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1665	$source_file_line_number++;
	1666	}
	1667	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1668
	1669	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1670	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1671	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1672
	1673	# If no translation exists for this chunk, show this, and move on
	1674	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1675	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	1676	next;
	1677	}
	1678
	1679	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	1680	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1681	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1682	}
	1683	print TARGET_FILE "\n";
[10019]	1684	}
[24627]	1685
[10019]	1686	close(TARGET_FILE);
	1687	}
	1688
	1689
[11321]	1690	# ==========================================================================================
	1691	# GREENSTONE XML FUNCTIONS
	1692
	1693	sub build_key_to_line_mapping_for_greenstone_xml
	1694	{
	1695	my (@file_lines) = @_;
[24627]	1696
[11321]	1697	my %chunk_key_to_line_mapping = ();
	1698	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1699	my $line = $file_lines[$i];
	1700	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1701
	1702	# Line contains a string to translate
	1703	if ($line =~ /^\s<Text id=\"(.?)\">/) {
	1704	my $chunk_key = $1;
	1705	$line =~ s/\s*$//; # Remove any nasty whitespace
	1706	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1707
	1708	# While there is still text of the string to go...
	1709	my $startline = $i;
	1710	while ($line !~ /<\/Text>$/) {
	1711	$i++;
	1712	if ($i == scalar(@file_lines)) {
	1713	&throw_fatal_error("Could not find end of string $chunk_key.");
	1714	}
	1715	$line = $file_lines[$i];
	1716	$line =~ s/\s*$//; # Remove any nasty whitespace
	1717	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1718	}
	1719
	1720	# Map from chunk key to line
	1721	if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
	1722	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1723	}
	1724	else {
	1725	&throw_fatal_error("Duplicate key $chunk_key.");
	1726	}
[11321]	1727	}
	1728	}
[24627]	1729
[11321]	1730	return %chunk_key_to_line_mapping;
	1731	}
	1732
	1733
	1734	sub import_chunk_from_greenstone_xml
	1735	{
	1736	my ($chunk_text) = @_;
[24627]	1737
[11321]	1738	# Simple: just remove the Text tags
[11447]	1739	$chunk_text =~ s/^\s<Text id=\"(.?)\">(\s*)//;
[12484]	1740	$chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
[11321]	1741	$chunk_text =~ s/<\/Text>$//;
[24627]	1742
[11321]	1743	return $chunk_text;
	1744	}
	1745
	1746
[12483]	1747	sub get_greenstone_xml_chunk_gti_comment
[11321]	1748	{
	1749	my ($chunk_text) = @_;
[24627]	1750
[11321]	1751	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1752	if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
[24627]	1753	return $1;
[11487]	1754	}
[24627]	1755
[11321]	1756	return undef;
	1757	}
	1758
	1759
	1760	sub is_greenstone_xml_chunk_automatically_translated
	1761	{
	1762	# No greenstone XML chunks are automatically translated
	1763	return 0;
	1764	}
	1765
	1766
	1767	sub write_translated_greenstone_xml
	1768	{
	1769	my $source_file = shift(@_); # Not used
	1770	my @source_file_lines = @{shift(@_)};
	1771	my $source_file_key_to_text_mapping = shift(@_);
	1772	my $target_file = shift(@_);
	1773	my @target_file_lines = @{shift(@_)}; # Not used
	1774	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1775	my $target_file_key_to_gti_comment_mapping = shift(@_);
[11321]	1776	my $target_language_code = shift(@_); # Not used
[24627]	1777
[11321]	1778	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	1779	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
	1780	my %source_file_line_to_key_mapping = ();
	1781	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1782	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[11321]	1783	}
[24627]	1784
[11321]	1785	# Write the new target file
	1786	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1787	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1788	&throw_fatal_error("Could not write target file $target_file_path.");
[11321]	1789	}
[24627]	1790
[11321]	1791	# Model the new target file on the source file, with the target file translations
	1792	my $source_file_line_number = 0;
	1793	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1794	# Fill in the gaps before this chunk starts
	1795	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1796	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1797	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1798	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1799	$source_file_line_number++;
	1800	}
	1801	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1802
	1803	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1804	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1805	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1806	$target_file_chunk_text =~ s/(\n)*$//g;
	1807
	1808	# If no translation exists for this chunk, show this, and move on
	1809	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1810	print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
	1811	next;
	1812	}
	1813
	1814	print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
	1815	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1816	my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1817	$chunk_gti_comment =~ s/^Updated //;
	1818	print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
	1819	}
	1820	print TARGET_FILE "\n";
[11321]	1821	}
[24627]	1822
[11321]	1823	# Fill in the end of the file
	1824	while ($source_file_line_number < scalar(@source_file_lines)) {
[24627]	1825	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1826	$source_file_line_number++;
[11321]	1827	}
[24627]	1828
[11321]	1829	close(TARGET_FILE);
	1830	}
	1831
	1832
[18460]	1833	# ==========================================================================================
	1834	# GREENSTONE3 FUNCTIONS
	1835
	1836	sub get_all_chunks_for_gs3
	1837	{
	1838	# The code of the target language (ensure it is lowercase)
	1839	my $target_language_code = lc(shift(@_));
	1840	my $translation_file_key = lc(shift(@_));
[24627]	1841
[18460]	1842	# Check that the necessary arguments were supplied
	1843	if (!$target_language_code) {
[24627]	1844	&throw_fatal_error("Missing command argument.");
[18460]	1845	}
[24627]	1846
[18460]	1847	# Get (and check) the translation configuration
	1848	# my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	1849
	1850	my %source_files_key_to_text_mapping = ();
	1851	my %target_files_key_to_text_mapping = ();
	1852	my %source_files_key_to_last_update_date_mapping = ();
	1853	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1854
[18460]	1855	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[24627]	1856
[18460]	1857	&log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
	1858	&log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
[24627]	1859
[18460]	1860	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1861	return $xml_response;
	1862	}
	1863
	1864
	1865	sub get_first_n_chunks_requiring_work_for_gs3
	1866	{
	1867	# The code of the target language (ensure it is lowercase)
	1868	my $target_language_code = lc(shift(@_));
	1869	# The key of the file to translate (ensure it is lowercase)
	1870	my $translation_file_key = lc(shift(@_));
	1871	# The number of chunks to return (defaults to one if not specified)
	1872	my $num_chunks_to_return = shift(@_) \|\| "1";
	1873
	1874	# Check that the necessary arguments were supplied
	1875	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	1876	&throw_fatal_error("Missing command argument.");
[18460]	1877	}
[28755]	1878
[18460]	1879	my %source_files_key_to_text_mapping = ();
	1880	my %target_files_key_to_text_mapping = ();
	1881	my %source_files_key_to_last_update_date_mapping = ();
	1882	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1883
[18460]	1884	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
[24627]	1885	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[18460]	1886
	1887	# Determine the target file chunks requiring translation
	1888	my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
	1889	# Determine the target file chunks requiring updating
	1890	my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1891	&log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
	1892	&log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
[24627]	1893
[18460]	1894	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
[24627]	1895	\@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
	1896	$num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1897	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1898
[18460]	1899	return $xml_response;
	1900	}
	1901
[25249]	1902	sub get_uptodate_chunks_for_gs3
	1903	{
	1904	# The code of the target language (ensure it is lowercase)
	1905	my $target_language_code = lc(shift(@_));
	1906	# The key of the file to translate (ensure it is lowercase)
	1907	my $translation_file_key = lc(shift(@_));
	1908	# The number of chunks to return (defaults to one if not specified)
	1909	my $num_chunks_to_return = shift(@_) \|\| "1";
	1910
	1911	# Check that the necessary arguments were supplied
	1912	if (!$target_language_code \|\| !$translation_file_key) {
	1913	&throw_fatal_error("Missing command argument.");
	1914	}
	1915
	1916	my %source_files_key_to_text_mapping = ();
	1917	my %target_files_key_to_text_mapping = ();
	1918	my %source_files_key_to_last_update_date_mapping = ();
	1919	my %target_files_key_to_last_update_date_mapping = ();
	1920
	1921	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1922	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1923
[18460]	1924
[25249]	1925	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1926	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	1927	my @uptodate_target_file_keys = ();
	1928	foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
	1929	my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
	1930	my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
	1931
	1932	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	1933
	1934	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	1935	# &log_message("Chunk with key $chunk_key needs updating.");
	1936	push(@uptodate_target_file_keys, $chunk_key);
	1937	}
	1938	}
[18460]	1939
[25249]	1940	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1941
	1942	return $xml_response;
	1943	}
	1944
	1945
[18460]	1946	sub build_gs3_configuration
	1947	{
	1948	my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
	1949	$source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
	1950
[28755]	1951	my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
[18460]	1952	my $translation_file_type = "resource_bundle";
[24627]	1953
[18460]	1954	foreach my $interface_file_key (@gs3_interface_files) {
[24627]	1955
	1956	&log_message("Greenstone 3 interface file: " . $interface_file_key);
	1957
	1958	# Parse the source language and target language files
	1959	my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
	1960	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	1961	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	1962	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	1963	my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	1964
	1965	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
	1966	my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	1967	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	1968	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	1969	my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	1970
	1971
	1972	# Filter out any automatically translated chunks
	1973	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	1974	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	1975	delete $source_file_key_to_line_mapping{$chunk_key};
	1976	delete $target_file_key_to_line_mapping{$chunk_key};
	1977	}
	1978	}
	1979
	1980	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	1981	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	1982
	1983	foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
	1984	my $global_chunk_key = "$interface_file_key.$chunk_key";
	1985	$source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
	1986	$source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
	1987
	1988	if (defined $target_file_key_to_text_mapping{$chunk_key}) {
	1989	$target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
	1990	$target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
	1991	}
	1992	}
[18460]	1993	}
	1994	}
	1995
	1996
	1997	sub write_translated_gs3interface
	1998	{
	1999	my $source_file_key_to_text_mapping = shift(@_);
	2000	my $target_file_key_to_text_mapping = shift(@_);
	2001	my $target_file_key_to_gti_comment_mapping = shift(@_);
	2002	my $target_language_code = shift(@_);
	2003
	2004	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
[24627]	2005
[18460]	2006	my %translated_interface_file_keys = ();
	2007	foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
[24627]	2008	$chunk_key =~ /^([^\.]+)?\.(.*)$/;
	2009	if (!defined $translated_interface_file_keys{$1}) {
	2010	&log_message("Updated interface file: " . $1);
	2011	$translated_interface_file_keys{$1}="";
	2012	}
[18460]	2013	}
	2014	&log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
	2015
	2016	my $source_file_directory = "greenstone3";
	2017
	2018	foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
[24627]	2019
	2020	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	2021	my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
	2022	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	2023	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
	2024	my %source_file_line_to_key_mapping = ();
	2025	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	2026	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[18460]	2027	}
[24627]	2028
	2029	# Write the new target file
	2030	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
	2031	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	2032	if (!open(TARGET_FILE, ">$target_file_path")) {
	2033	&throw_fatal_error("Could not write target file $target_file_path.");
	2034	}
	2035
	2036	# Model the new target file on the source file, with the target file translations
	2037	my $source_file_line_number = 0;
	2038	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
	2039	# Fill in the gaps before this chunk starts
	2040	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	2041	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	2042	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	2043	print TARGET_FILE $source_file_lines[$source_file_line_number];
	2044	$source_file_line_number++;
	2045	}
	2046	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	2047
	2048	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	2049	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2050	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
	2051	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} \|\| "";
	2052
	2053	# If no translation exists for this chunk, show this, and move on
	2054	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	2055	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	2056	next;
	2057	}
	2058
	2059	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	2060	if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
	2061	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
	2062	}
	2063	print TARGET_FILE "\n";
	2064	}
	2065
	2066	close(TARGET_FILE);
[18460]	2067	}
	2068	}
	2069
[10019]	2070	&main(@ARGV);

Note: See TracBrowser for help on using the repository browser.

Download in other formats: