Context Navigation

source: main/trunk/greenstone2/bin/script/gti.pl@ 29415

Last change on this file since 29415 was 29415, checked in by ak19, 9 years ago
There was no option for downloading language files for GS3. But only if a filepath to that is provided does the other link show up that allows downloading the spreadsheet for offline translation. For now adding in a version that zips up all the GS3 important interface properties files in English, and those in the translator's language where these exist.
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 90.3 KB

Rev	Line
[10019]	1	#!/usr/bin/perl -w
	2
	3	###########################################################################
	4	#
	5	# gti.pl
	6	#
	7	# A component of the Greenstone digital library software
	8	# from the New Zealand Digital Library Project at the
	9	# University of Waikato, New Zealand.
	10	#
	11	# Copyright (C) 2005 New Zealand Digital Library Project
	12	#
	13	# This program is free software; you can redistribute it and/or modify
	14	# it under the terms of the GNU General Public License as published by
	15	# the Free Software Foundation; either version 2 of the License, or
	16	# (at your option) any later version.
	17	#
	18	# This program is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	# GNU General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU General Public License
	24	# along with this program; if not, write to the Free Software
	25	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	26	#
	27	###########################################################################
	28
	29
	30	BEGIN {
	31	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
	32	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
	33	}
	34
[24627]	35
[10019]	36	use iso639;
	37	use strict;
	38	use util;
[29415]	39	use FileUtils;
[10019]	40
	41	my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
	42	my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
[28976]	43	my $source_language_code = "en"; # This is non-negotiable
[10019]	44
	45	my $gti_translation_files =
[24627]	46	[ # Greenstone macrofiles
	47	{ 'key' => "coredm",
[10019]	48	'file_type' => "macrofile",
	49	'source_file' => "macros/english.dm",
[11224]	50	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[24627]	51
	52	{ 'key' => "auxdm",
[10019]	53	'file_type' => "macrofile",
	54	'source_file' => "macros/english2.dm",
[11224]	55	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
[10019]	56
[29411]	57	#{ 'key' => "paperspastdm",
	58	# 'file_type' => "macrofile",
	59	# 'source_file' => "macros/paperspast-english.dm",
	60	# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[25285]	61
[24627]	62	# GLI dictionary
	63	{ 'key' => "glidict",
[13946]	64	'file_type' => "resource_bundle",
	65	'source_file' => "gli/classes/dictionary.properties",
	66	'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
[10114]	67
[24627]	68	# GLI help
	69	{ 'key' => "glihelp",
[13946]	70	'file_type' => "greenstone_xml",
	71	'source_file' => "gli/help/en/help.xml",
	72	'target_file' => "gli/help/{target_language_code}/help.xml" },
[11602]	73
[24627]	74	# Greenstone Perl modules
	75	{ 'key' => "perlmodules",
[10019]	76	'file_type' => "resource_bundle",
[11634]	77	'source_file' => "perllib/strings.properties",
	78	'target_file' => "perllib/strings_{target_language_code}.properties" },
[10019]	79
[28337]	80	# Greenstone Installer interface
	81	{ 'key' => "gsinstaller",
	82	'file_type' => "resource_bundle",
	83	'source_file' => "gsinstaller/LanguagePack.properties",
	84	'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
	85
[24627]	86	# Greenstone tutorial exercises
	87	# { 'key' => "tutorials",
	88	# 'file_type' => "greenstone_xml",
	89	# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
	90	# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
[11366]	91
[24627]	92	# new Greenstone.org
	93	{ 'key' => "greenorg",
	94	'file_type' => "resource_bundle",
	95	'source_file' => "greenstoneorg/website/classes/Gsc.properties",
	96	'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
[29411]	97	},
	98
[29412]	99	# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
[29413]	100	# check it out as greenstone3
[29411]	101	{ 'key' => "gs3interface",
[29415]	102	'file_type' => "resource_bundle",
	103	'source_file' => "greenstone3",
	104	'target_file' => "greenstone3"
[24627]	105	}
	106	];
[10019]	107
[29411]	108	my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_default2", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
[11026]	109
[26547]	110	# Auxilliary GS3 interface files. This list is not used at present
	111	# Combine with above list if generating translation spreadsheet for all interface files
	112	my @gs3_aux_interface_files = ("GATEServices", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2", "QBRWebServicesHelp", "Visualizer");
[18460]	113
[26547]	114	# Not: i18n, log4j
	115
[10019]	116	sub main
	117	{
	118	# Get the command to process, and any arguments
	119	my $gti_command = shift(@_);
	120	my @gti_command_arguments = @_;
[18460]	121	my $module = $_[1];
[26544]	122
	123	# for GS3, set gsdl_root_dir to GSDL3HOME
[29411]	124	#if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
	125	#if($ENV{'GSDL3SRCHOME'}) {
	126	# $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
	127	# $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
	128	#}
	129	#}
[24627]	130
[10019]	131	# Open the GTI log file for appending, or write to STDERR if that fails
	132	if (!open(GTI_LOG, ">>$gti_log_file")) {
[24627]	133	open(GTI_LOG, ">&STDERR");
[10019]	134	}
[24627]	135
[10019]	136	# Log the command that launched this script
	137	&log_message("Command: $0 @ARGV");
[24627]	138
[10019]	139	# Check that a command was supplied
	140	if (!$gti_command) {
[24627]	141	&throw_fatal_error("Missing command.");
[18460]	142	}
[24627]	143
[10019]	144	# Process the command
[13948]	145	if ($gti_command =~ /^get-all-chunks$/i) {
[24627]	146	# Check that GS3 interface is the target
[26544]	147	if ($module eq "gs3interface") {
[24627]	148	print &get_all_chunks_for_gs3(@gti_command_arguments);
	149	} else {
	150	print &get_all_chunks(@gti_command_arguments);
	151	}
[13948]	152	}
[24627]	153	elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
	154	if ($module eq "gs3interface") {
	155	print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
	156	} else {
	157	print &get_first_n_chunks_requiring_work(@gti_command_arguments);
	158	}
[10019]	159	}
[25249]	160	elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
	161	if ($module eq "gs3interface") {
	162	print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
	163	} else {
	164	print &get_uptodate_chunks(@gti_command_arguments);
	165	}
	166	}
[24627]	167	elsif ($gti_command =~ /^get-language-status$/i) {
	168	print &get_language_status(@gti_command_arguments);
[10019]	169	}
[24627]	170	elsif ($gti_command =~ /^search-chunks$/i) {
	171	print &search_chunks(@gti_command_arguments);
[10019]	172	}
[24627]	173	elsif ($gti_command =~ /^submit-translations$/i) {
	174	# This command cannot produce any output since it reads input
	175	&submit_translations(@gti_command_arguments);
[10019]	176	}
[24627]	177	elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
	178	# This command cannot produce any output since it reads input
	179	&create_glihelp_zip_file(@gti_command_arguments);
[18460]	180	}
[24627]	181	else {
	182	# The command was not recognized
	183	&throw_fatal_error("Unknown command \"$gti_command\".");
	184	}
[10019]	185	}
	186
	187
	188	sub throw_fatal_error
	189	{
	190	my $error_message = shift(@_);
[24627]	191
[10019]	192	# Write an XML error response
	193	print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	194	print "<GTIResponse>\n";
	195	print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
	196	print "</GTIResponse>\n";
[24627]	197
[10019]	198	# Log the error message, then die
	199	&log_message("Error: $error_message");
	200	die "\n";
	201	}
	202
	203
	204	sub log_message
	205	{
	206	my $log_message = shift(@_);
	207	print GTI_LOG time() . " -- " . $log_message . "\n";
	208	}
	209
	210
[13948]	211	sub get_all_chunks
	212	{
	213	# The code of the target language (ensure it is lowercase)
	214	my $target_language_code = lc(shift(@_));
	215	# The key of the file to translate (ensure it is lowercase)
	216	my $translation_file_key = lc(shift(@_));
[24627]	217
[13948]	218	# Check that the necessary arguments were supplied
	219	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	220	&throw_fatal_error("Missing command argument.");
[13948]	221	}
[24627]	222
[13948]	223	# Get (and check) the translation configuration
	224	my ($source_file, $target_file, $translation_file_type)
	225	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	226
[13948]	227	# Parse the source language and target language files
	228	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	229	my @source_file_lines = &read_file_lines($source_file_path);
	230	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	231
	232	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	233	my @target_file_lines = &read_file_lines($target_file_path);
	234	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	235
[13948]	236	# Filter out any automatically translated chunks
	237	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	238	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	239	delete $source_file_key_to_line_mapping{$chunk_key};
	240	delete $target_file_key_to_line_mapping{$chunk_key};
	241	}
[13948]	242	}
[24627]	243
[13948]	244	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	245	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	246	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	247	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	248
[13948]	249	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	250	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[24627]	251
[18460]	252	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
[13948]	253
	254	return $xml_response;
	255	}
	256
	257
[25249]	258	sub get_uptodate_chunks
	259	{
	260	# The code of the target language (ensure it is lowercase)
	261	my $target_language_code = lc(shift(@_));
	262	# The key of the file to translate (ensure it is lowercase)
	263	my $translation_file_key = lc(shift(@_));
	264
	265	# Check that the necessary arguments were supplied
	266	if (!$target_language_code \|\| !$translation_file_key) {
	267	&throw_fatal_error("Missing command argument.");
	268	}
	269
	270	# Get (and check) the translation configuration
	271	my ($source_file, $target_file, $translation_file_type)
	272	= &get_translation_configuration($target_language_code, $translation_file_key);
	273
	274	# Parse the source language and target language files
	275	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	276	my @source_file_lines = &read_file_lines($source_file_path);
	277	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	278
	279	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	280	my @target_file_lines = &read_file_lines($target_file_path);
	281	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	282
	283	# Filter out any automatically translated chunks
	284	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	285	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	286	delete $source_file_key_to_line_mapping{$chunk_key};
	287	delete $target_file_key_to_line_mapping{$chunk_key};
	288	}
	289	}
	290
	291	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	292	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	293	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	294	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	295
	296	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	297	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	298
	299
	300	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	301	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	302	my @uptodate_target_file_keys = ();
	303	foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
	304	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
	305	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
	306
	307	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	308
	309	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	310	# &log_message("Chunk with key $chunk_key needs updating.");
	311	push(@uptodate_target_file_keys, $chunk_key);
	312	}
	313	}
	314
	315	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	316
	317	return $xml_response;
	318	}
	319
	320
[10019]	321	sub get_first_n_chunks_requiring_work
	322	{
	323	# The code of the target language (ensure it is lowercase)
	324	my $target_language_code = lc(shift(@_));
	325	# The key of the file to translate (ensure it is lowercase)
	326	my $translation_file_key = lc(shift(@_));
	327	# The number of chunks to return (defaults to one if not specified)
	328	my $num_chunks_to_return = shift(@_) \|\| "1";
[24627]	329
[10019]	330	# Check that the necessary arguments were supplied
	331	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	332	&throw_fatal_error("Missing command argument.");
[10019]	333	}
[24627]	334
[10019]	335	# Get (and check) the translation configuration
	336	my ($source_file, $target_file, $translation_file_type)
	337	= &get_translation_configuration($target_language_code, $translation_file_key);
[28755]	338
[10019]	339	# Parse the source language and target language files
	340	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	341	my @source_file_lines = &read_file_lines($source_file_path);
	342	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
[14258]	343
[10019]	344	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	345	my @target_file_lines = &read_file_lines($target_file_path);
	346	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	347
[10019]	348	# Filter out any automatically translated chunks
	349	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	350	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	351	delete $source_file_key_to_line_mapping{$chunk_key};
	352	delete $target_file_key_to_line_mapping{$chunk_key};
	353	}
[10019]	354	}
[24627]	355
[10019]	356	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	357	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	358	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	359	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	360
[10019]	361	# Determine the target file chunks requiring translation
	362	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	363	&log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
[24627]	364
[10019]	365	# Determine the target file chunks requiring updating
	366	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	367	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	368	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	369	&log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
[24627]	370
[18460]	371	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	372
[10040]	373	return $xml_response;
[10019]	374	}
	375
	376
	377	sub get_language_status
	378	{
	379	# The code of the target language (ensure it is lowercase)
	380	my $target_language_code = lc(shift(@_));
[24627]	381
[10019]	382	# Check that the necessary arguments were supplied
	383	if (!$target_language_code) {
[24627]	384	&throw_fatal_error("Missing command argument.");
[10019]	385	}
[24627]	386
[10040]	387	# Form an XML response to the command
	388	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	389	$xml_response .= "<GTIResponse>\n";
	390	$xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
[24627]	391
[18460]	392	foreach my $translation_file (@$gti_translation_files) {
[24627]	393	my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
	394	my $target_file_name = "";
	395
	396	if ($translation_file->{'key'} eq "gs3interface") {
	397	my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
	398	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
	399
	400	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	401	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	402
	403	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	404	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	405	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	406	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
[18460]	407	}
[24627]	408	else {
	409	# Get (and check) the translation configuration
	410	my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
	411	$target_file_name = $target_file;
	412
	413	# Parse the source language and target language files
	414	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	415	my @source_file_lines = &read_file_lines($source_file_path);
	416	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	417
	418	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	419	my @target_file_lines = &read_file_lines($target_file_path);
	420	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	421
	422	# Filter out any automatically translated chunks
	423	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	424	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	425	delete $source_file_key_to_line_mapping{$chunk_key};
	426	delete $target_file_key_to_line_mapping{$chunk_key};
	427	}
	428	}
	429
	430	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	431	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	432
	433	# Determine the target file chunks requiring translation
	434	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	435
	436	# Determine the target file chunks requiring updating
	437	my @target_file_keys_requiring_updating = ();
	438	if (-e $target_file_path) {
	439	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	440	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	441	@target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	442	}
	443
	444	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	445	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	446	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	447	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
	448	}
	449
	450	&log_message("Status of " . $translation_file->{'key'});
	451	&log_message("Number of source chunks: " . $num_source_chunks);
	452	&log_message("Number of target chunks: " . $num_target_chunks);
	453	&log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
	454	&log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
	455
	456	$xml_response .= " <TranslationFile"
[10019]	457	. " key=\"" . $translation_file->{'key'} . "\""
[18460]	458	. " target_file_path=\"" . $target_file_name . "\""
	459	. " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
	460	. " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
	461	. " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
[10019]	462	}
[24627]	463
[10040]	464	$xml_response .= " </LanguageStatus>\n";
[24627]	465
[10040]	466	$xml_response .= "</GTIResponse>\n";
	467	return $xml_response;
[10019]	468	}
	469
	470
	471	sub search_chunks
	472	{
	473	# The code of the target language (ensure it is lowercase)
	474	my $target_language_code = lc(shift(@_));
	475	# The key of the file to translate (ensure it is lowercase)
	476	my $translation_file_key = lc(shift(@_));
	477	# The query string
[10020]	478	my $query_string = join(' ', @_);
[24627]	479
[10019]	480	# Check that the necessary arguments were supplied
	481	if (!$target_language_code \|\| !$translation_file_key \|\| !$query_string) {
[24627]	482	&throw_fatal_error("Missing command argument.");
[10019]	483	}
[24627]	484
[18460]	485	my ($source_file, $target_file, $translation_file_type) = ();
	486	my %source_file_key_to_text_mapping = ();
	487	my %target_file_key_to_text_mapping = ();
	488
	489
	490	if ($translation_file_key ne "gs3interface") {
[24627]	491	# Get (and check) the translation configuration
	492	($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	493
	494	# Parse the source language and target language files
	495	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	496	my @source_file_lines = &read_file_lines($source_file_path);
	497	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	498
	499	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	500	my @target_file_lines = &read_file_lines($target_file_path);
	501	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	502
	503	# Filter out any automatically translated chunks
	504	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	505	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	506	delete $source_file_key_to_line_mapping{$chunk_key};
	507	delete $target_file_key_to_line_mapping{$chunk_key};
	508	}
	509	}
	510
	511	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	512	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[10019]	513	}
[18460]	514	else {
[24627]	515	# Not needed in this case
	516	my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
	517	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	518	\%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
[18460]	519	}
[24627]	520
[10019]	521	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	522	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	523
[10019]	524	# Determine the target file chunks matching the query
	525	my @target_file_keys_matching_query = ();
	526	foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
[24627]	527	my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
	528	if ($target_file_text =~ /$query_string/i) {
	529	# &log_message("Chunk with key $chunk_key matches query.");
	530	push(@target_file_keys_matching_query, $chunk_key);
	531	}
[10019]	532	}
[24627]	533
[10040]	534	# Form an XML response to the command
	535	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	536	$xml_response .= "<GTIResponse>\n";
[24627]	537
[10040]	538	$xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
[10019]	539	foreach my $chunk_key (@target_file_keys_matching_query) {
[24627]	540	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
	541
	542	$xml_response .= " <Chunk key=\"$chunk_key\">\n";
	543	$xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
	544	$xml_response .= " </Chunk>\n";
[10019]	545	}
[10040]	546	$xml_response .= " </ChunksMatchingQuery>\n";
[24627]	547
[10040]	548	$xml_response .= "</GTIResponse>\n";
	549	return $xml_response;
[10019]	550	}
	551
	552
[10050]	553	sub submit_translations
[10019]	554	{
	555	# The code of the target language (ensure it is lowercase)
	556	my $target_language_code = lc(shift(@_));
	557	# The key of the file to translate (ensure it is lowercase)
	558	my $translation_file_key = lc(shift(@_));
[12484]	559	# The username of the translation submitter
	560	my $submitter_username = shift(@_);
[10027]	561	# Whether to submit a target chunk even if it hasn't changed
	562	my $force_submission_flag = shift(@_);
[24627]	563
[10019]	564	# Check that the necessary arguments were supplied
[12484]	565	if (!$target_language_code \|\| !$translation_file_key \|\| !$submitter_username) {
[24627]	566	&log_message("Fatal error (but cannot be thrown): Missing command argument.");
	567	die "\n";
[10019]	568	}
[18460]	569
	570	my %source_file_key_to_text_mapping = ();
	571	my %source_file_key_to_gti_comment_mapping = ();
	572	my %target_file_key_to_text_mapping = ();
	573	my %target_file_key_to_gti_comment_mapping = ();
[24627]	574
[18460]	575	my (@source_file_lines, @target_file_lines) = ();
	576	my ($source_file, $target_file, $translation_file_type);
[24627]	577
[18460]	578
	579	if ($translation_file_key ne "gs3interface") {
[24627]	580	# Get (and check) the translation configuration
	581	($source_file, $target_file, $translation_file_type)
[18460]	582	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	583
	584	# Parse the source language and target language files
	585	@source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	586	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	587	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	588	%source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	589
	590	@target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	591	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	592	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	593	%target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[18460]	594	}
	595	else {
[24627]	596	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	597	\%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
[18460]	598	}
[10019]	599	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	600	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	601
[10019]	602	# Submission date
	603	my $day = (localtime)[3];
	604	my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
	605	my $year = (localtime)[5] + 1900;
	606	my $submission_date = "$day-$month-$year";
[24627]	607
[10019]	608	open(SUBMISSION, "-");
	609	my @submission_lines = <SUBMISSION>;
	610	close(SUBMISSION);
[24627]	611
[10019]	612	# Remove any nasty carriage returns
[18460]	613	# &log_message("Submission:");
[10019]	614	foreach my $submission_line (@submission_lines) {
[24627]	615	$submission_line =~ s/\r$//;
	616	#&log_message(" $submission_line");
[10019]	617	}
[24627]	618
[10019]	619	my %source_file_key_to_submission_mapping = ();
	620	my %target_file_key_to_submission_mapping = ();
	621	for (my $i = 0; $i < scalar(@submission_lines); $i++) {
[24627]	622	# Read source file part of submission
	623	if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
	624	my $chunk_key = $1;
	625
	626	# Read the source file text
	627	my $source_file_chunk_text = "";
	628	$i++;
	629	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
	630	$source_file_chunk_text .= $submission_lines[$i];
	631	$i++;
	632	}
	633	$source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	634	$source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
	635
	636	#&log_message("Source file key: $chunk_key");
	637	#&log_message("Source file text: $source_file_chunk_text");
	638	$source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
	639	}
	640
	641	# Read target file part of submission
	642	if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
	643	my $chunk_key = $1;
	644
	645	# Read the target file text
	646	my $target_file_chunk_text = "";
	647	$i++;
	648	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
	649	$target_file_chunk_text .= $submission_lines[$i];
	650	$i++;
	651	}
	652	$target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	653	$target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
	654
	655	#&log_message("Target file key: $chunk_key");
	656	#&log_message("Target file text: $target_file_chunk_text");
	657	$target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
	658	}
[10019]	659	}
[24627]	660
[10019]	661	# -----------------------------------------
	662	# Validate the translation submissions
	663	# -----------------------------------------
[24627]	664
[10019]	665	# Check that the translations are valid
	666	foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
[24627]	667	# Make sure the submitted chunk still exists in the source file
	668	if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
	669	&log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
	670	delete $source_file_key_to_submission_mapping{$chunk_key};
	671	delete $target_file_key_to_submission_mapping{$chunk_key};
	672	next;
	673	}
	674
	675	# Make sure the submitted source chunk matches the source file chunk
[28518]	676	if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
	677	#if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
[28503]	678	#print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
	679	#print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
	680
[24627]	681	&log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
	682	&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
	683	&log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
	684	delete $source_file_key_to_submission_mapping{$chunk_key};
	685	delete $target_file_key_to_submission_mapping{$chunk_key};
	686	next;
	687	}
[10027]	688	}
[24627]	689
[10027]	690	# Apply the submitted translations
	691	foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
[24627]	692	# Only apply the submission if it is a change, unless -force_submission has been specified
	693	if ($force_submission_flag \|\| !defined($target_file_key_to_text_mapping{$chunk_key}) \|\| $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
	694	$target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
	695	$target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
	696	}
[10019]	697	}
[18460]	698
	699	if ($translation_file_key ne "gs3interface") {
[24627]	700	eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	701	} else {
[24627]	702	eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	703	}
	704	}
[10019]	705
[18460]	706
	707	sub create_glihelp_zip_file
	708	{
	709	my $target_language_code = shift(@_);
	710	my $translation_file_key = "glihelp";
	711
	712	&log_message("Creating GLI Help zip file for $target_language_code");
[24627]	713
[18460]	714	my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
	715
	716	my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
[28977]	717	my $oldclasspath = $classpath;
[18460]	718	if ( ! -e $classpath) {
[28977]	719	$classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
	720	}
	721	if ( ! -e $classpath) {
	722	&throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
	723	}
	724
[28991]	725
	726	my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
	727	my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
	728	my $os = $^O;
	729	my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
	730	my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
	731	$classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
	732
[18460]	733	my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
	734	$gli_help_directory = &util::filename_cat($gli_help_directory, "help");
	735
	736	my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
	737	if ( ! -e $gen_many_html_xsl_filepath) {
[24627]	738	&throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
[18460]	739	}
[24627]	740
[18460]	741	my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
	742	my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
	743
	744	my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
	745	$target_file_directory = $target_file_directory."/";
[24627]	746
[18460]	747	my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
[28991]	748
	749	# if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
	750	my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
	751	if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
	752
	753	# copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
	754	my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
[28992]	755	my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
[28991]	756	&FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
	757
	758	# In gli/help/<newlang>/help.xml, replace all occurrences of
	759	# <Text id="1">This text in en will be removed for new langcode</Text>
	760	# with <!-- Missing translation: 1 -->
	761	open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
	762	my $help_xml_contents;
	763	# Read in the entire contents of the file in one hit
	764	sysread(FIN, $help_xml_contents, -s FIN);
	765	close(FIN);
[24627]	766
[28991]	767	$help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
	768
	769	open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
	770	print FOUT $help_xml_contents;
	771	close(FOUT);
	772	}
	773
[24627]	774	my $perl_exec = &util::get_perl_exec();
	775	my $java_exec = "java";
	776	if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
	777	$java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
	778	}
[28991]	779
	780	#my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	781	my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	782	#&throw_fatal_error("RAN gti command: $cmd");
[18460]	783	my $response = `$cmd`;
[28991]	784
	785	#$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
	786	$cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
[18460]	787	$response = `$cmd`;
[28991]	788
	789	# create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
	790	my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
	791	if(!&FileUtils::directoryExists($tmpdir)) {
	792	&FileUtils::makeDirectory($tmpdir);
	793	}
[28977]	794	#my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
[28991]	795	my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
[18460]	796	$cmd = "zip -rj $zip_file_path $target_file_directory -i \.htm \.xml";
[28991]	797
[18460]	798	$response = `$cmd`;
[10019]	799	}
	800
	801
	802	sub get_translation_configuration
	803	{
	804	# Get the code of the target language
	805	my $target_language_code = shift(@_);
	806	# Get the key of the file to translate
	807	my $translation_file_key = shift(@_);
[24627]	808
[10019]	809	# Read the translation data from the gti.cfg file
	810	my ($source_file, $target_file, $translation_file_type) =
	811	&get_translation_data_for($target_language_code, $translation_file_key);
[24627]	812
[10019]	813	# Check that the file to translate is defined in the gti.cfg file
	814	if (!$source_file \|\| !$target_file \|\| !$translation_file_type) {
[24627]	815	&throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
[10019]	816	}
[24627]	817
[10019]	818	# Check that the source file exists
	819	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	820	if (!-e $source_file_path) {
[24627]	821	&throw_fatal_error("Source file $source_file_path does not exist.");
[10019]	822	}
[24627]	823
[10019]	824	# Check that the source file is up to date
[11104]	825	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[10019]	826	# unless ($translation_file_is_not_in_cvs) {
[14258]	827	#my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
[24627]	828	my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
[10019]	829	if ($source_file_cvs_status =~ /^C /) {
	830	&throw_fatal_error("Source file $source_file_path conflicts with the repository.");
	831	}
	832	if ($source_file_cvs_status =~ /^M /) {
	833	&throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
	834	}
	835	# }
[24627]	836
[10019]	837	return ($source_file, $target_file, $translation_file_type);
	838	}
	839
	840
	841	sub get_translation_data_for
	842	{
	843	my ($target_language_code, $translation_file_key) = @_;
[24627]	844
[10019]	845	foreach my $translation_file (@$gti_translation_files) {
[24627]	846	# If this isn't the correct translation file, move onto the next one
	847	next if ($translation_file_key ne $translation_file->{'key'});
	848
	849	# Resolve the target language file
	850	my $target_language_file = $translation_file->{'target_file'};
	851	if ($target_language_file =~ /(\{.+\;.+\})/) {
	852	my $unresolved_target_language_file_part = $1;
	853
	854	# Check for a special case for the target language code
	855	if ($unresolved_target_language_file_part =~ /(\{\|\;)$target_language_code:([^\;]+)(\;\|\})/) {
	856	my $resolved_target_language_file_part = $2;
	857	$target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
	858	}
	859	# Otherwise use the last part as the default value
	860	else {
	861	my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
	862	$target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
[10019]	863	}
[10041]	864	}
[24627]	865
[10041]	866	# Resolve instances of {iso_639_1_target_language_name}
	867	my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
	868	$iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
	869	$target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
[24627]	870
[10041]	871	# Resolve instances of {target_language_code}
	872	$target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
[24627]	873
[10041]	874	return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
[24627]	875	}
[10019]	876
[24627]	877	return ();
[10019]	878	}
	879
	880
	881	sub read_file_lines
	882	{
	883	my ($file_path) = @_;
[24627]	884
[10019]	885	if (!open(FILE_IN, "<$file_path")) {
[24627]	886	&log_message("Note: Could not open file $file_path.");
	887	return ();
[10019]	888	}
	889	my @file_lines = <FILE_IN>;
	890	close(FILE_IN);
[24627]	891
[10019]	892	return @file_lines;
	893	}
	894
	895
	896	sub build_key_to_line_mapping
	897	{
	898	my ($file_lines, $translation_file_type) = @_;
	899	eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
	900	}
	901
	902
	903	sub build_key_to_text_mapping
	904	{
	905	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	906
[10019]	907	my %key_to_text_mapping = ();
	908	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	909	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	910	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	911
	912	my $chunk_text = @$file_lines[$chunk_starting_line];
	913	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	914	$chunk_text .= @$file_lines[$l];
	915	}
	916
	917	# Map from chunk key to text
	918	eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
[10019]	919	}
[24627]	920
[10019]	921	return %key_to_text_mapping;
	922	}
	923
	924
	925	sub build_key_to_last_update_date_mapping
	926	{
	927	my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	928
[10019]	929	# If the files aren't in CVS then we can't tell anything about what needs updating
	930	# return () if ($translation_file_is_not_in_cvs);
[24627]	931
[10019]	932	# Build a mapping from key to CVS date
	933	# Need to be careful with this mapping because the chunk keys won't necessarily all be valid
	934	my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
[24627]	935
[10019]	936	# Build a mapping from key to comment date
[12483]	937	my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
[24627]	938
[10019]	939	# Build a mapping from key to last update date (the latter of the CVS date and comment date)
	940	my %key_to_last_update_date_mapping = ();
	941	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	942	# Use the CVS date as a starting point
	943	my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
	944	$key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
	945
	946	# If a comment date exists and it is after the CVS date, use that instead
[14258]	947	# need to convert the comment date format to SVN format
[24627]	948	my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
	949	if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
	950	my $chunk_comment_date = $1;
	951	if ((!defined($chunk_cvs_date) \|\| &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
	952	$key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
	953	}
	954	}
[10019]	955	}
[24627]	956
[10019]	957	return %key_to_last_update_date_mapping;
	958	}
	959
	960
	961	sub build_key_to_cvs_date_mapping
	962	{
	963	my ($filename, $translation_file_type) = @_;
[24627]	964
	965	# Use SVN to annotate each line of the file with the date it was last edited
[10019]	966	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[24627]	967	my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
[14258]	968
[10019]	969	my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
[24627]	970
[10019]	971	my @cvs_annotated_file_lines_date = ();
	972	foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
[24627]	973	# Extract the date from the SVN annotation at the front
	974	# svn format : 2007-07-16
[14258]	975	$cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
	976
	977	push(@cvs_annotated_file_lines_date, $1);
	978
	979	# trim extra date information in svn annotation format
	980	# 15:42:49 +1200 (Wed, 21 Jun 2006)
	981	$cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
	982	}
	983
[10019]	984	# Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
	985	my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
[24627]	986
[10019]	987	my %key_to_cvs_date_mapping = ();
	988	foreach my $chunk_key (keys(%key_to_line_mapping)) {
[24627]	989	my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
	990	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
	991
	992	# Find the date this chunk was last edited, from the CVS annotation
	993	my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
	994	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	995	if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
	996	# This part of the chunk has been updated more recently
	997	$chunk_date = $cvs_annotated_file_lines_date[$l];
	998
	999	}
	1000	}
	1001
	1002	# Map from chunk key to CVS date
	1003	$key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
[10019]	1004	}
[24627]	1005
[10019]	1006	return %key_to_cvs_date_mapping;
	1007	}
	1008
	1009
[12483]	1010	sub build_key_to_gti_comment_mapping
[10019]	1011	{
	1012	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	1013
[12483]	1014	my %key_to_gti_comment_mapping = ();
[10019]	1015	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	1016	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	1017	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	1018
	1019	my $chunk_text = @$file_lines[$chunk_starting_line];
	1020	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	1021	$chunk_text .= @$file_lines[$l];
	1022	}
	1023
	1024	# Map from chunk key to GTI comment
	1025	my $chunk_gti_comment;
	1026	eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
	1027	$key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
[10019]	1028	}
[24627]	1029
[12483]	1030	return %key_to_gti_comment_mapping;
[10019]	1031	}
	1032
	1033
	1034	sub determine_chunks_requiring_translation
	1035	{
	1036	my $source_file_key_to_text_mapping = shift(@_);
	1037	my $target_file_key_to_text_mapping = shift(@_);
[24627]	1038
[10019]	1039	# Chunks needing translation are those in the source file with no translation in the target file
	1040	my @target_file_keys_requiring_translation = ();
	1041	foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
[24627]	1042	if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
	1043	# &log_message("Chunk with key $chunk_key needs translating.");
	1044	push(@target_file_keys_requiring_translation, $chunk_key);
	1045	}
[10019]	1046	}
[24627]	1047
[10019]	1048	return @target_file_keys_requiring_translation;
	1049	}
	1050
	1051
	1052	sub determine_chunks_requiring_updating
	1053	{
	1054	my $source_file_key_to_last_update_date_mapping = shift(@_);
	1055	my $target_file_key_to_last_update_date_mapping = shift(@_);
[24627]	1056
[10019]	1057	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1058	my @target_file_keys_requiring_updating = ();
	1059	foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
[24627]	1060	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
	1061	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
[14258]	1062
	1063	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
[24627]	1064
[14258]	1065	if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
[24627]	1066	# &log_message("Chunk with key $chunk_key needs updating.");
	1067	push(@target_file_keys_requiring_updating, $chunk_key);
	1068	}
[10019]	1069	}
[24627]	1070
[10019]	1071	return @target_file_keys_requiring_updating;
	1072	}
	1073
	1074
	1075	sub is_chunk_automatically_translated
	1076	{
	1077	my ($chunk_key, $translation_file_type) = @_;
	1078	eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
	1079	}
	1080
	1081
	1082	sub make_text_xml_safe
	1083	{
	1084	my $text = shift(@_);
[11452]	1085	$text =~ s/\&/\&amp\;/g;
	1086	$text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
	1087	$text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
[11498]	1088	$text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
	1089	$text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
[10019]	1090	$text =~ s/</\&lt\;/g;
	1091	$text =~ s/>/\&gt\;/g;
	1092	return $text;
	1093	}
	1094
	1095
[11448]	1096	sub unmake_text_xml_safe
	1097	{
	1098	my $text = shift(@_);
	1099	$text =~ s/\&lt\;/</g;
	1100	$text =~ s/\&gt\;/>/g;
	1101	$text =~ s/\&amp\;/\&/g;
	1102	return $text;
	1103	}
	1104
	1105
[10019]	1106	# Returns 1 if $date1 is after $date2, 0 otherwise
[14258]	1107	sub is_date_after_cvs
[10019]	1108	{
	1109	my ($date1, $date2) = @_;
	1110	my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
[24627]	1111	"Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
	1112
[14258]	1113	if(!defined $date1) {
	1114	return 1;
	1115	}
[24627]	1116
[10019]	1117	my @date1parts = split(/-/, $date1);
	1118	my @date2parts = split(/-/, $date2);
[24627]	1119
[10019]	1120	# Compare year - nasty because we have rolled over into a new century
	1121	my $year1 = $date1parts[2];
	1122	if ($year1 < 80) {
[14258]	1123	$year1 += 2000;
[10019]	1124	}
	1125	my $year2 = $date2parts[2];
	1126	if ($year2 < 80) {
[14258]	1127	$year2 += 2000;
[10019]	1128	}
[24627]	1129
[10019]	1130	# Compare year
	1131	if ($year1 > $year2) {
[24627]	1132	return 1;
[10019]	1133	}
	1134	elsif ($year1 == $year2) {
[24627]	1135	# Year is the same, so compare month
	1136	if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
	1137	return 1;
	1138	}
	1139	elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
	1140	# Month is the same, so compare day
	1141	if ($date1parts[0] > $date2parts[0]) {
	1142	return 1;
	1143	}
	1144	}
[10019]	1145	}
[24627]	1146
[10019]	1147	return 0;
	1148	}
	1149
[14258]	1150	sub is_date_after
	1151	{
	1152	my ($date1, $date2) = @_;
	1153
	1154	if(!defined $date1) {
[24627]	1155	return 1;
[14258]	1156	}
	1157	if(!defined $date2) {
[24627]	1158	return 0;
[14258]	1159	}
	1160
	1161	# 16-Aug-2006
	1162	if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1163	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1164	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1165	$date1=$3 . "-" . $months{$2} . "-" . $1;
	1166	# print "** converted date1: $date1\n";
[14258]	1167	}
	1168	if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1169	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1170	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1171	$date2=$3 . "-" . $months{$2} . "-" . $1;
	1172	# print "** converted date2: $date2\n";
[14258]	1173	}
	1174
	1175
	1176	# 2006-08-16
	1177	my @date1parts = split(/-/, $date1);
	1178	my @date2parts = split(/-/, $date2);
	1179
	1180	# Compare year
	1181	if ($date1parts[0] > $date2parts[0]) {
[24627]	1182	return 1;
[14258]	1183	}
	1184	elsif ($date1parts[0] == $date2parts[0]) {
[24627]	1185	# Year is the same, so compare month
	1186	if ($date1parts[1] > $date2parts[1]) {
	1187	return 1;
	1188	}
	1189	elsif ($date1parts[1] == $date2parts[1]) {
	1190	# Month is the same, so compare day
	1191	if ($date1parts[2] > $date2parts[2]) {
	1192	return 1;
	1193	}
	1194	}
[14258]	1195	}
	1196
	1197	return 0;
	1198	}
[10019]	1199
[14258]	1200
[18460]	1201	sub create_xml_response_for_chunks_requiring_work
	1202	{
	1203	my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
[24627]	1204
[18460]	1205	# Form an XML response to the command
	1206	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1207	$xml_response .= "<GTIResponse>\n";
	1208	$xml_response .= " <TranslationFile"
	1209	. " key=\"" . $translation_file_key . "\""
	1210	. " target_file_path=\"" . $target_file . "\""
	1211	. " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
	1212	. " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
	1213	. " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
[24627]	1214
[18460]	1215	# Do chunks requiring translation first
	1216	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
[24627]	1217	$xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
[18460]	1218	}
	1219	else {
[24627]	1220	$xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1221	}
[24627]	1222
[18460]	1223	my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
	1224	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1225	last if ($num_chunks_to_return == 0);
	1226
	1227	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1228	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1229
	1230	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1231	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1232	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1233	$xml_response .= " </Chunk>\n";
	1234
	1235	$num_chunks_to_return--;
	1236	}
[18460]	1237
	1238	$xml_response .= " </ChunksRequiringTranslation>\n";
[24627]	1239
[18460]	1240	# Then do chunks requiring updating
	1241	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
[24627]	1242	$xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
[18460]	1243	}
	1244	else {
[24627]	1245	$xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1246	}
[24627]	1247
[18460]	1248	# foreach my $chunk_key (@target_file_keys_requiring_updating) {
	1249	@sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
	1250	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1251	last if ($num_chunks_to_return == 0);
	1252
	1253	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1254	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1255	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1256	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1257
	1258	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1259	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1260	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1261	$xml_response .= " </Chunk>\n";
	1262
	1263	$num_chunks_to_return--;
[18460]	1264	}
[24627]	1265
[18460]	1266	$xml_response .= " </ChunksRequiringUpdating>\n";
[24627]	1267
[18460]	1268	$xml_response .= "</GTIResponse>\n";
[24627]	1269
[18460]	1270	return $xml_response;
	1271	}
	1272
[25249]	1273	sub create_xml_response_for_uptodate_chunks
	1274	{
	1275	my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
	1276
	1277	# Form an XML response to the command
	1278	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1279	$xml_response .= "<GTIResponse>\n";
	1280	$xml_response .= " <TranslationFile"
	1281	. " key=\"" . $translation_file_key . "\""
	1282	. " target_file_path=\"" . $target_file . "\""
	1283	. " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
	1284
	1285
	1286	# Then do chunks requiring updating
	1287	$xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
	1288
	1289
	1290	# foreach my $chunk_key (@uptodate_target_file_keys) {
	1291	my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
	1292	foreach my $chunk_key (@sorted_chunk_keys) {
	1293
	1294	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1295	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1296	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1297	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1298
	1299	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1300	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1301	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1302	$xml_response .= " </Chunk>\n";
[18460]	1303
[25249]	1304	}
	1305
	1306	$xml_response .= " </UptodateChunks>\n";
	1307
	1308	$xml_response .= "</GTIResponse>\n";
	1309
	1310	return $xml_response;
	1311	}
	1312
[18460]	1313	sub create_xml_response_for_all_chunks
	1314	{
	1315	my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
[24627]	1316
[18460]	1317	# Form an XML response to the command
	1318	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1319	$xml_response .= "<GTIResponse>\n";
	1320	$xml_response .= " <TranslationFile"
	1321	. " key=\"" . $translation_file_key . "\""
	1322	. " target_file_path=\"" . $target_file . "\"\/>\n";
	1323
	1324	# Do all the chunks
	1325	$xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
[24627]	1326
[18460]	1327	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
	1328	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1329	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1330	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
	1331
	1332	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1333	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1334	if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
	1335	my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1336	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
	1337	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1338	}
	1339	else {
	1340	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1341	}
	1342
	1343	$xml_response .= " </Chunk>\n";
[18460]	1344	}
	1345	$xml_response .= " </Chunks>\n";
	1346
	1347	$xml_response .= "</GTIResponse>\n";
	1348	return $xml_response;
	1349	}
	1350
	1351
	1352
[10019]	1353	# ==========================================================================================
	1354	# MACROFILE FUNCTIONS
	1355
	1356	sub build_key_to_line_mapping_for_macrofile
	1357	{
	1358	my (@file_lines) = @_;
[24627]	1359
[10019]	1360	my $macro_package;
	1361	my %chunk_key_to_line_mapping = ();
	1362	# Process the contents of the file, line by line
	1363	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1364	my $line = $file_lines[$i];
	1365	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1366
	1367	# Check if a new package is being defined
	1368	if ($line =~ m/^package\s+(.+)/) {
	1369	$macro_package = $1;
[10041]	1370	}
[24627]	1371
	1372	# Line contains a macro name
	1373	elsif ($line =~ m/^(_\w+_)/) {
	1374	my $macro_key = $1;
	1375	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1376
	1377	# While there is still text of the macro to go...
	1378	my $startline = $i;
	1379	while ($line !~ /\}$/) {
	1380	$i++;
	1381	if ($i == scalar(@file_lines)) {
	1382	&throw_fatal_error("Could not find end of macro $macro_key.");
	1383	}
	1384	$line = $file_lines[$i];
	1385	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1386	}
	1387
[10019]	1388	# The chunk key consists of the package name and the macro key
	1389	my $chunk_key = $macro_package . "." . $macro_key;
	1390	# Map from chunk key to line
	1391	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1392	}
[24627]	1393
[10019]	1394	# Icon: line in format ## "image text" ## image_type ## macro_name ##
	1395	elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
[24627]	1396	# The chunk key consists of package name and macro key
	1397	my $chunk_key = $macro_package . "." . $1;
	1398	# Map from chunk key to line
	1399	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1400	}
	1401	}
[10019]	1402
[24627]	1403	return %chunk_key_to_line_mapping;
[10019]	1404	}
	1405
	1406
	1407	sub import_chunk_from_macrofile
	1408	{
	1409	my ($chunk_text) = @_;
[24627]	1410
[10019]	1411	# Is this an icon macro??
	1412	if ($chunk_text =~ /^\#\# (.*)/) {
[24627]	1413	# Extract image macro text
	1414	$chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
	1415	$chunk_text = $1;
	1416
[10019]	1417	# Remove enclosing quotes
	1418	$chunk_text =~ s/^\"//;
	1419	$chunk_text =~ s/\"$//;
[24627]	1420	}
[10019]	1421
[24627]	1422	# No, so it must be a text macro
	1423	else {
[10019]	1424	# Remove macro key
	1425	$chunk_text =~ s/^_([^_]+)_(\s*)//;
[24627]	1426
[10019]	1427	# Remove language specifier
	1428	$chunk_text =~ s/^\[l=.\](\s)//;
[24627]	1429
[10019]	1430	# Remove braces enclosing text
	1431	$chunk_text =~ s/^{(\s)((.\|\n))}(\s)(\#.+\s)?/$2/;
[24627]	1432	}
[10019]	1433
[24627]	1434	return $chunk_text;
[10019]	1435	}
	1436
	1437
[12483]	1438	sub get_macrofile_chunk_gti_comment
[10019]	1439	{
	1440	my ($chunk_text) = @_;
[24627]	1441
[10019]	1442	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1443	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1444	return $1;
	1445	}
[10019]	1446
[24627]	1447	return undef;
[10019]	1448	}
	1449
	1450
	1451	sub is_macrofile_chunk_automatically_translated
	1452	{
	1453	my ($chunk_key) = @_;
[24627]	1454
[10019]	1455	# The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
	1456	if ($chunk_key =~ /\._(httpicon\|width\|height)/) {
[24627]	1457	return 1;
[10019]	1458	}
[24627]	1459
[10019]	1460	return 0;
	1461	}
	1462
	1463
	1464	# Use the source file to generate a target file that is formatted the same
	1465	sub write_translated_macrofile
	1466	{
	1467	my $source_file = shift(@_); # Not used
[11321]	1468	my @source_file_lines = @{shift(@_)};
[10019]	1469	my $source_file_key_to_text_mapping = shift(@_);
	1470	my $target_file = shift(@_);
[11321]	1471	my @target_file_lines = @{shift(@_)};
[10019]	1472	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1473	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1474	my $target_language_code = shift(@_);
[24627]	1475
[10126]	1476	# Build a mapping from source file line to chunk key
[11321]	1477	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
[10019]	1478	my %source_file_line_to_key_mapping = ();
	1479	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1480	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1481	}
[10126]	1482	my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
	1483	my $source_file_line_number = 0;
[24627]	1484
[10126]	1485	# Build a mapping from target file line to chunk key
[11321]	1486	my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
[10126]	1487	my %target_file_line_to_key_mapping = ();
	1488	foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
[24627]	1489	$target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10126]	1490	}
	1491	my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
[24627]	1492
[10019]	1493	# Write the new target file
	1494	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1495	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1496	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1497	}
[24627]	1498
[10126]	1499	# Use the header from the target file, to keep language and author information
	1500	if (scalar(@target_file_line_keys) > 0) {
[24627]	1501	my $target_file_line_number = 0;
	1502	my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
	1503	while ($target_file_line_number < $target_file_chunk_starting_line_number) {
	1504	my $target_file_line = $target_file_lines[$target_file_line_number];
	1505	last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
	1506	print TARGET_FILE $target_file_line;
	1507	$target_file_line_number++;
	1508	}
	1509
	1510	$source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
[10126]	1511	}
[24627]	1512
[10019]	1513	# Model the new target file on the source file, with the target file translations
[10126]	1514	foreach my $line_key (@source_file_line_keys) {
[24627]	1515	# Fill in the gaps before this chunk starts
	1516	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1517	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1518	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1519	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1520	$source_file_line_number++;
	1521	}
	1522	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1523
	1524	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1525	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1526	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1527
	1528	my $macrofile_key = $chunk_key;
	1529	$macrofile_key =~ s/^(.+?)\.//;
	1530
	1531	# If no translation exists for this chunk, show this, and move on
	1532	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1533	print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
	1534	next;
	1535	}
	1536
	1537	# Grab the source chunk text
	1538	my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
	1539	for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
	1540	$source_file_chunk .= $source_file_lines[$l];
	1541	}
	1542
	1543	# Is this an icon macro??
	1544	if ($source_file_chunk =~ /^\#\# (.*)/) {
	1545	# Escape any newline and question mark characters so the source text is replaced correctly
	1546	$source_file_chunk_text =~ s/\\/\\\\/g;
[10019]	1547	$source_file_chunk_text =~ s/\?/\\\?/g;
[24627]	1548
[10019]	1549	# Build the new target chunk from the source chunk
	1550	my $target_file_chunk = $source_file_chunk;
	1551	$target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
[10050]	1552	$target_file_chunk =~ s/(\s)*$//;
[10019]	1553	print TARGET_FILE "$target_file_chunk";
	1554	}
[24627]	1555
[10019]	1556	# No, it is just a normal text macro
	1557	else {
	1558	print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
	1559	}
[24627]	1560
[12483]	1561	# Add the "updated" comment, if one exists
	1562	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1563	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
[10050]	1564	}
	1565	print TARGET_FILE "\n";
[24627]	1566	}
[10019]	1567
[24627]	1568	close(TARGET_FILE);
[10019]	1569	}
	1570
	1571
	1572	sub sort_by_line
	1573	{
	1574	return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
	1575	}
	1576
	1577
	1578	# ==========================================================================================
	1579	# RESOURCE BUNDLE FUNCTIONS
	1580
	1581	sub build_key_to_line_mapping_for_resource_bundle
	1582	{
	1583	my (@file_lines) = @_;
[24627]	1584
[11321]	1585	my %chunk_key_to_line_mapping = ();
[10019]	1586	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1587	my $line = $file_lines[$i];
	1588	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1589
	1590	# Line contains a dictionary string
	1591	if ($line =~ /^(\S+?)[:\|=](.*)$/) {
	1592	my $chunk_key = $1;
	1593
	1594	# Map from chunk key to line
	1595	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1596	}
[10019]	1597	}
[24627]	1598
[11321]	1599	return %chunk_key_to_line_mapping;
[10019]	1600	}
	1601
	1602
	1603	sub import_chunk_from_resource_bundle
	1604	{
	1605	my ($chunk_text) = @_;
[24627]	1606
[10019]	1607	# Simple: just remove string key
[18460]	1608	$chunk_text =~ s/^(\S+?)[:\|=](\s*)//;
[10019]	1609	$chunk_text =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
[12484]	1610	$chunk_text =~ s/(\s)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.)\s*$//i;
[24627]	1611
[10019]	1612	return $chunk_text;
	1613	}
	1614
	1615
[12483]	1616	sub get_resource_bundle_chunk_gti_comment
[10019]	1617	{
	1618	my ($chunk_text) = @_;
[24627]	1619
[10019]	1620	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1621	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1622	return $1;
	1623	}
[10019]	1624
[24627]	1625	return undef;
[10019]	1626	}
	1627
	1628
	1629	sub is_resource_bundle_chunk_automatically_translated
	1630	{
	1631	# No resource bundle chunks are automatically translated
	1632	return 0;
	1633	}
	1634
	1635
	1636	sub write_translated_resource_bundle
	1637	{
	1638	my $source_file = shift(@_); # Not used
[11321]	1639	my @source_file_lines = @{shift(@_)};
[10019]	1640	my $source_file_key_to_text_mapping = shift(@_);
	1641	my $target_file = shift(@_);
[11321]	1642	my @target_file_lines = @{shift(@_)}; # Not used
[10019]	1643	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1644	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1645	my $target_language_code = shift(@_); # Not used
[24627]	1646
[10019]	1647	# Build a mapping from chunk key to source file line, and from source file line to chunk key
[11321]	1648	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
[10019]	1649	my %source_file_line_to_key_mapping = ();
	1650	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1651	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1652	}
[24627]	1653
[10019]	1654	# Write the new target file
	1655	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1656	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1657	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1658	}
[24627]	1659
[10019]	1660	# Model the new target file on the source file, with the target file translations
[10091]	1661	my $source_file_line_number = 0;
[10019]	1662	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1663	# Fill in the gaps before this chunk starts
	1664	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1665	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1666	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1667	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1668	$source_file_line_number++;
	1669	}
	1670	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1671
	1672	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1673	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1674	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1675
	1676	# If no translation exists for this chunk, show this, and move on
	1677	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1678	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	1679	next;
	1680	}
	1681
	1682	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	1683	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1684	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1685	}
	1686	print TARGET_FILE "\n";
[10019]	1687	}
[24627]	1688
[10019]	1689	close(TARGET_FILE);
	1690	}
	1691
	1692
[11321]	1693	# ==========================================================================================
	1694	# GREENSTONE XML FUNCTIONS
	1695
	1696	sub build_key_to_line_mapping_for_greenstone_xml
	1697	{
	1698	my (@file_lines) = @_;
[24627]	1699
[11321]	1700	my %chunk_key_to_line_mapping = ();
	1701	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1702	my $line = $file_lines[$i];
	1703	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1704
	1705	# Line contains a string to translate
	1706	if ($line =~ /^\s<Text id=\"(.?)\">/) {
	1707	my $chunk_key = $1;
	1708	$line =~ s/\s*$//; # Remove any nasty whitespace
	1709	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1710
	1711	# While there is still text of the string to go...
	1712	my $startline = $i;
	1713	while ($line !~ /<\/Text>$/) {
	1714	$i++;
	1715	if ($i == scalar(@file_lines)) {
	1716	&throw_fatal_error("Could not find end of string $chunk_key.");
	1717	}
	1718	$line = $file_lines[$i];
	1719	$line =~ s/\s*$//; # Remove any nasty whitespace
	1720	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1721	}
	1722
	1723	# Map from chunk key to line
	1724	if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
	1725	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1726	}
	1727	else {
	1728	&throw_fatal_error("Duplicate key $chunk_key.");
	1729	}
[11321]	1730	}
	1731	}
[24627]	1732
[11321]	1733	return %chunk_key_to_line_mapping;
	1734	}
	1735
	1736
	1737	sub import_chunk_from_greenstone_xml
	1738	{
	1739	my ($chunk_text) = @_;
[24627]	1740
[11321]	1741	# Simple: just remove the Text tags
[11447]	1742	$chunk_text =~ s/^\s<Text id=\"(.?)\">(\s*)//;
[12484]	1743	$chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
[11321]	1744	$chunk_text =~ s/<\/Text>$//;
[24627]	1745
[11321]	1746	return $chunk_text;
	1747	}
	1748
	1749
[12483]	1750	sub get_greenstone_xml_chunk_gti_comment
[11321]	1751	{
	1752	my ($chunk_text) = @_;
[24627]	1753
[11321]	1754	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1755	if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
[24627]	1756	return $1;
[11487]	1757	}
[24627]	1758
[11321]	1759	return undef;
	1760	}
	1761
	1762
	1763	sub is_greenstone_xml_chunk_automatically_translated
	1764	{
	1765	# No greenstone XML chunks are automatically translated
	1766	return 0;
	1767	}
	1768
	1769
	1770	sub write_translated_greenstone_xml
	1771	{
	1772	my $source_file = shift(@_); # Not used
	1773	my @source_file_lines = @{shift(@_)};
	1774	my $source_file_key_to_text_mapping = shift(@_);
	1775	my $target_file = shift(@_);
	1776	my @target_file_lines = @{shift(@_)}; # Not used
	1777	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1778	my $target_file_key_to_gti_comment_mapping = shift(@_);
[11321]	1779	my $target_language_code = shift(@_); # Not used
[24627]	1780
[11321]	1781	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	1782	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
	1783	my %source_file_line_to_key_mapping = ();
	1784	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1785	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[11321]	1786	}
[24627]	1787
[11321]	1788	# Write the new target file
	1789	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1790	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1791	&throw_fatal_error("Could not write target file $target_file_path.");
[11321]	1792	}
[24627]	1793
[11321]	1794	# Model the new target file on the source file, with the target file translations
	1795	my $source_file_line_number = 0;
	1796	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1797	# Fill in the gaps before this chunk starts
	1798	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1799	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1800	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1801	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1802	$source_file_line_number++;
	1803	}
	1804	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1805
	1806	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1807	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1808	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1809	$target_file_chunk_text =~ s/(\n)*$//g;
	1810
	1811	# If no translation exists for this chunk, show this, and move on
	1812	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1813	print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
	1814	next;
	1815	}
	1816
	1817	print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
	1818	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1819	my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1820	$chunk_gti_comment =~ s/^Updated //;
	1821	print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
	1822	}
	1823	print TARGET_FILE "\n";
[11321]	1824	}
[24627]	1825
[11321]	1826	# Fill in the end of the file
	1827	while ($source_file_line_number < scalar(@source_file_lines)) {
[24627]	1828	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1829	$source_file_line_number++;
[11321]	1830	}
[24627]	1831
[11321]	1832	close(TARGET_FILE);
	1833	}
	1834
	1835
[18460]	1836	# ==========================================================================================
	1837	# GREENSTONE3 FUNCTIONS
	1838
	1839	sub get_all_chunks_for_gs3
	1840	{
	1841	# The code of the target language (ensure it is lowercase)
	1842	my $target_language_code = lc(shift(@_));
	1843	my $translation_file_key = lc(shift(@_));
[24627]	1844
[18460]	1845	# Check that the necessary arguments were supplied
	1846	if (!$target_language_code) {
[24627]	1847	&throw_fatal_error("Missing command argument.");
[18460]	1848	}
[24627]	1849
[18460]	1850	# Get (and check) the translation configuration
	1851	# my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	1852
	1853	my %source_files_key_to_text_mapping = ();
	1854	my %target_files_key_to_text_mapping = ();
	1855	my %source_files_key_to_last_update_date_mapping = ();
	1856	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1857
[18460]	1858	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[24627]	1859
[18460]	1860	&log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
	1861	&log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
[24627]	1862
[18460]	1863	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1864	return $xml_response;
	1865	}
	1866
	1867
	1868	sub get_first_n_chunks_requiring_work_for_gs3
	1869	{
	1870	# The code of the target language (ensure it is lowercase)
	1871	my $target_language_code = lc(shift(@_));
	1872	# The key of the file to translate (ensure it is lowercase)
	1873	my $translation_file_key = lc(shift(@_));
	1874	# The number of chunks to return (defaults to one if not specified)
	1875	my $num_chunks_to_return = shift(@_) \|\| "1";
	1876
	1877	# Check that the necessary arguments were supplied
	1878	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	1879	&throw_fatal_error("Missing command argument.");
[18460]	1880	}
[28755]	1881
[18460]	1882	my %source_files_key_to_text_mapping = ();
	1883	my %target_files_key_to_text_mapping = ();
	1884	my %source_files_key_to_last_update_date_mapping = ();
	1885	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1886
[18460]	1887	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
[24627]	1888	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[18460]	1889
	1890	# Determine the target file chunks requiring translation
	1891	my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
	1892	# Determine the target file chunks requiring updating
	1893	my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1894	&log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
	1895	&log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
[29415]	1896
	1897	my $download_target_filepath = "";
	1898
	1899
	1900	# **** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION **** #
	1901
	1902	# if there is no copy of the language files for download, there's also no link to the spreadsheet
	1903	# for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
	1904	# interface *.properties files,and link to that zip as the file for offline translation.
	1905	# Selecting only properties files for English and the language they're working on (if the last exists)
	1906
	1907	# tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
	1908	# will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
	1909
	1910	my $sourcedir = "greenstone3";
	1911	my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
	1912	my $tar_cmd = "tar -cvzf $zip";
	1913
	1914
	1915	# store cur dir and cd to gsdlhome to generate the correct path in the zip file
	1916	my $curdir = `pwd`;
	1917	chdir $gsdl_root_directory;
	1918
	1919	my $filelisting = "";
	1920	foreach my $interface_file (@gs3_interface_files) {
	1921
	1922	my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
	1923	my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
[24627]	1924
[29415]	1925	$filelisting = "$filelisting $source_filepath";
	1926	if(&FileUtils::fileExists($target_filepath)) {
	1927	$filelisting = "$filelisting $target_filepath";
	1928	}
	1929	}
	1930	$tar_cmd .= " $filelisting";
	1931
	1932	# tar command will overwrite the previous version, but want to check we've created it
	1933	if(&FileUtils::fileExists($zip)) {
	1934	&FileUtils::removeFiles($zip);
	1935	}
	1936
	1937	#my $tar_result = system($tar_cmd); # works but then interface breaks
	1938	`$tar_cmd`;
	1939	my $tar_result = $?;
	1940
	1941	if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
	1942	$download_target_filepath = $zip;
	1943	} else {
	1944	&log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
	1945	}
	1946
	1947	# change back to original working directory (cgi-bin/linux probably)
	1948	chdir $curdir;
	1949
	1950	# ************ END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 *********** #
	1951
	1952
	1953	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
[24627]	1954	\@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
	1955	$num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1956	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1957
[18460]	1958	return $xml_response;
	1959	}
	1960
[25249]	1961	sub get_uptodate_chunks_for_gs3
	1962	{
	1963	# The code of the target language (ensure it is lowercase)
	1964	my $target_language_code = lc(shift(@_));
	1965	# The key of the file to translate (ensure it is lowercase)
	1966	my $translation_file_key = lc(shift(@_));
	1967	# The number of chunks to return (defaults to one if not specified)
	1968	my $num_chunks_to_return = shift(@_) \|\| "1";
	1969
	1970	# Check that the necessary arguments were supplied
	1971	if (!$target_language_code \|\| !$translation_file_key) {
	1972	&throw_fatal_error("Missing command argument.");
	1973	}
	1974
	1975	my %source_files_key_to_text_mapping = ();
	1976	my %target_files_key_to_text_mapping = ();
	1977	my %source_files_key_to_last_update_date_mapping = ();
	1978	my %target_files_key_to_last_update_date_mapping = ();
	1979
	1980	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1981	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1982
[18460]	1983
[25249]	1984	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1985	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	1986	my @uptodate_target_file_keys = ();
	1987	foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
	1988	my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
	1989	my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
	1990
	1991	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	1992
	1993	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	1994	# &log_message("Chunk with key $chunk_key needs updating.");
	1995	push(@uptodate_target_file_keys, $chunk_key);
	1996	}
	1997	}
[18460]	1998
[25249]	1999	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	2000
	2001	return $xml_response;
	2002	}
	2003
	2004
[18460]	2005	sub build_gs3_configuration
	2006	{
	2007	my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
	2008	$source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
	2009
[28755]	2010	my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
[18460]	2011	my $translation_file_type = "resource_bundle";
[24627]	2012
[18460]	2013	foreach my $interface_file_key (@gs3_interface_files) {
[24627]	2014
	2015	&log_message("Greenstone 3 interface file: " . $interface_file_key);
	2016
	2017	# Parse the source language and target language files
	2018	my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
	2019	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	2020	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	2021	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	2022	my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	2023
	2024	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
	2025	my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	2026	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	2027	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	2028	my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	2029
	2030
	2031	# Filter out any automatically translated chunks
	2032	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	2033	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	2034	delete $source_file_key_to_line_mapping{$chunk_key};
	2035	delete $target_file_key_to_line_mapping{$chunk_key};
	2036	}
	2037	}
	2038
	2039	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	2040	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	2041
	2042	foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
	2043	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2044	$source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
	2045	$source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
	2046
	2047	if (defined $target_file_key_to_text_mapping{$chunk_key}) {
	2048	$target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
	2049	$target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
	2050	}
	2051	}
[18460]	2052	}
	2053	}
	2054
	2055
	2056	sub write_translated_gs3interface
	2057	{
	2058	my $source_file_key_to_text_mapping = shift(@_);
	2059	my $target_file_key_to_text_mapping = shift(@_);
	2060	my $target_file_key_to_gti_comment_mapping = shift(@_);
	2061	my $target_language_code = shift(@_);
	2062
	2063	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
[24627]	2064
[18460]	2065	my %translated_interface_file_keys = ();
	2066	foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
[24627]	2067	$chunk_key =~ /^([^\.]+)?\.(.*)$/;
	2068	if (!defined $translated_interface_file_keys{$1}) {
	2069	&log_message("Updated interface file: " . $1);
	2070	$translated_interface_file_keys{$1}="";
	2071	}
[18460]	2072	}
	2073	&log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
	2074
	2075	my $source_file_directory = "greenstone3";
	2076
	2077	foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
[24627]	2078
	2079	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	2080	my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
	2081	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	2082	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
	2083	my %source_file_line_to_key_mapping = ();
	2084	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	2085	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[18460]	2086	}
[24627]	2087
	2088	# Write the new target file
	2089	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
	2090	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	2091	if (!open(TARGET_FILE, ">$target_file_path")) {
	2092	&throw_fatal_error("Could not write target file $target_file_path.");
	2093	}
	2094
	2095	# Model the new target file on the source file, with the target file translations
	2096	my $source_file_line_number = 0;
	2097	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
	2098	# Fill in the gaps before this chunk starts
	2099	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	2100	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	2101	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	2102	print TARGET_FILE $source_file_lines[$source_file_line_number];
	2103	$source_file_line_number++;
	2104	}
	2105	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	2106
	2107	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	2108	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2109	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
	2110	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} \|\| "";
	2111
	2112	# If no translation exists for this chunk, show this, and move on
	2113	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	2114	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	2115	next;
	2116	}
	2117
	2118	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	2119	if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
	2120	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
	2121	}
	2122	print TARGET_FILE "\n";
	2123	}
	2124
	2125	close(TARGET_FILE);
[18460]	2126	}
	2127	}
	2128
[10019]	2129	&main(@ARGV);

Note: See TracBrowser for help on using the repository browser.

Download in other formats: