Context Navigation

source: main/trunk/greenstone2/bin/script/gti.pl@ 28518

Last change on this file since 28518 was 28518, checked in by ak19, 10 years ago
Undoing the change I made in an earlier commit, since that breaks the general case even though it works correctly in a special case. Need to think more on this.
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 85.0 KB

Rev	Line
[10019]	1	#!/usr/bin/perl -w
	2
	3	###########################################################################
	4	#
	5	# gti.pl
	6	#
	7	# A component of the Greenstone digital library software
	8	# from the New Zealand Digital Library Project at the
	9	# University of Waikato, New Zealand.
	10	#
	11	# Copyright (C) 2005 New Zealand Digital Library Project
	12	#
	13	# This program is free software; you can redistribute it and/or modify
	14	# it under the terms of the GNU General Public License as published by
	15	# the Free Software Foundation; either version 2 of the License, or
	16	# (at your option) any later version.
	17	#
	18	# This program is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	# GNU General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU General Public License
	24	# along with this program; if not, write to the Free Software
	25	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	26	#
	27	###########################################################################
	28
	29
	30	BEGIN {
	31	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
	32	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
	33	}
	34
[24627]	35
[10019]	36	use iso639;
	37	use strict;
	38	use util;
	39
	40	my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
	41	my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
	42	my $source_language_code = "en"; # This is non-negiotable
	43
	44	my $gti_translation_files =
[24627]	45	[ # Greenstone macrofiles
	46	{ 'key' => "coredm",
[10019]	47	'file_type' => "macrofile",
	48	'source_file' => "macros/english.dm",
[11224]	49	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[24627]	50
	51	{ 'key' => "auxdm",
[10019]	52	'file_type' => "macrofile",
	53	'source_file' => "macros/english2.dm",
[11224]	54	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
[10019]	55
[25285]	56	{ 'key' => "paperspastdm",
	57	'file_type' => "macrofile",
	58	'source_file' => "macros/paperspast-english.dm",
	59	'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
	60
[24627]	61	# GLI dictionary
	62	{ 'key' => "glidict",
[13946]	63	'file_type' => "resource_bundle",
	64	'source_file' => "gli/classes/dictionary.properties",
	65	'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
[10114]	66
[24627]	67	# GLI help
	68	{ 'key' => "glihelp",
[13946]	69	'file_type' => "greenstone_xml",
	70	'source_file' => "gli/help/en/help.xml",
	71	'target_file' => "gli/help/{target_language_code}/help.xml" },
[11602]	72
[24627]	73	# Greenstone Perl modules
	74	{ 'key' => "perlmodules",
[10019]	75	'file_type' => "resource_bundle",
[11634]	76	'source_file' => "perllib/strings.properties",
	77	'target_file' => "perllib/strings_{target_language_code}.properties" },
[10019]	78
[28337]	79	# Greenstone Installer interface
	80	{ 'key' => "gsinstaller",
	81	'file_type' => "resource_bundle",
	82	'source_file' => "gsinstaller/LanguagePack.properties",
	83	'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
	84
[24627]	85	# Greenstone tutorial exercises
	86	# { 'key' => "tutorials",
	87	# 'file_type' => "greenstone_xml",
	88	# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
	89	# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
[11366]	90
[24627]	91	# new Greenstone.org
	92	{ 'key' => "greenorg",
	93	'file_type' => "resource_bundle",
	94	'source_file' => "greenstoneorg/website/classes/Gsc.properties",
	95	'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
	96	}
[18460]	97
[24627]	98	# { 'key' => "gs3interface",
	99	#'file_type' => "resource_bundle",
	100	#'source_file' => "greenstone3",
	101	#'target_file' => "greenstone3"
	102	#}
	103	];
[10019]	104
[26547]	105	my @gs3_interface_files = ("AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "Authentication", "CrossCollectionSearch", "GS2Construct", "GS2LuceneSearch", "interface_default", "interface_default2", "IViaSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "metadata_names", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
[11026]	106
[26547]	107	# Auxilliary GS3 interface files. This list is not used at present
	108	# Combine with above list if generating translation spreadsheet for all interface files
	109	my @gs3_aux_interface_files = ("GATEServices", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2", "QBRWebServicesHelp", "Visualizer");
[18460]	110
[26547]	111	# Not: i18n, log4j
	112
[10019]	113	sub main
	114	{
	115	# Get the command to process, and any arguments
	116	my $gti_command = shift(@_);
	117	my @gti_command_arguments = @_;
[18460]	118	my $module = $_[1];
[26544]	119
	120	# for GS3, set gsdl_root_dir to GSDL3HOME
	121	if($module eq "gs3interface"){
	122	$gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
	123	$gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
	124	}
[24627]	125
[10019]	126	# Open the GTI log file for appending, or write to STDERR if that fails
	127	if (!open(GTI_LOG, ">>$gti_log_file")) {
[24627]	128	open(GTI_LOG, ">&STDERR");
[10019]	129	}
[24627]	130
[10019]	131	# Log the command that launched this script
	132	&log_message("Command: $0 @ARGV");
[24627]	133
[10019]	134	# Check that a command was supplied
	135	if (!$gti_command) {
[24627]	136	&throw_fatal_error("Missing command.");
[18460]	137	}
[24627]	138
[10019]	139	# Process the command
[13948]	140	if ($gti_command =~ /^get-all-chunks$/i) {
[24627]	141	# Check that GS3 interface is the target
[26544]	142	if ($module eq "gs3interface") {
[24627]	143	print &get_all_chunks_for_gs3(@gti_command_arguments);
	144	} else {
	145	print &get_all_chunks(@gti_command_arguments);
	146	}
[13948]	147	}
[24627]	148	elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
	149	if ($module eq "gs3interface") {
	150	print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
	151	} else {
	152	print &get_first_n_chunks_requiring_work(@gti_command_arguments);
	153	}
[10019]	154	}
[25249]	155	elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
	156	if ($module eq "gs3interface") {
	157	print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
	158	} else {
	159	print &get_uptodate_chunks(@gti_command_arguments);
	160	}
	161	}
[24627]	162	elsif ($gti_command =~ /^get-language-status$/i) {
	163	print &get_language_status(@gti_command_arguments);
[10019]	164	}
[24627]	165	elsif ($gti_command =~ /^search-chunks$/i) {
	166	print &search_chunks(@gti_command_arguments);
[10019]	167	}
[24627]	168	elsif ($gti_command =~ /^submit-translations$/i) {
	169	# This command cannot produce any output since it reads input
	170	&submit_translations(@gti_command_arguments);
[10019]	171	}
[24627]	172	elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
	173	# This command cannot produce any output since it reads input
	174	&create_glihelp_zip_file(@gti_command_arguments);
[18460]	175	}
[24627]	176	else {
	177	# The command was not recognized
	178	&throw_fatal_error("Unknown command \"$gti_command\".");
	179	}
[10019]	180	}
	181
	182
	183	sub throw_fatal_error
	184	{
	185	my $error_message = shift(@_);
[24627]	186
[10019]	187	# Write an XML error response
	188	print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	189	print "<GTIResponse>\n";
	190	print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
	191	print "</GTIResponse>\n";
[24627]	192
[10019]	193	# Log the error message, then die
	194	&log_message("Error: $error_message");
	195	die "\n";
	196	}
	197
	198
	199	sub log_message
	200	{
	201	my $log_message = shift(@_);
	202	print GTI_LOG time() . " -- " . $log_message . "\n";
	203	}
	204
	205
[13948]	206	sub get_all_chunks
	207	{
	208	# The code of the target language (ensure it is lowercase)
	209	my $target_language_code = lc(shift(@_));
	210	# The key of the file to translate (ensure it is lowercase)
	211	my $translation_file_key = lc(shift(@_));
[24627]	212
[13948]	213	# Check that the necessary arguments were supplied
	214	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	215	&throw_fatal_error("Missing command argument.");
[13948]	216	}
[24627]	217
[13948]	218	# Get (and check) the translation configuration
	219	my ($source_file, $target_file, $translation_file_type)
	220	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	221
[13948]	222	# Parse the source language and target language files
	223	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	224	my @source_file_lines = &read_file_lines($source_file_path);
	225	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	226
	227	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	228	my @target_file_lines = &read_file_lines($target_file_path);
	229	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	230
[13948]	231	# Filter out any automatically translated chunks
	232	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	233	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	234	delete $source_file_key_to_line_mapping{$chunk_key};
	235	delete $target_file_key_to_line_mapping{$chunk_key};
	236	}
[13948]	237	}
[24627]	238
[13948]	239	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	240	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	241	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	242	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	243
[13948]	244	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	245	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[24627]	246
[18460]	247	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
[13948]	248
	249	return $xml_response;
	250	}
	251
	252
[25249]	253	sub get_uptodate_chunks
	254	{
	255	# The code of the target language (ensure it is lowercase)
	256	my $target_language_code = lc(shift(@_));
	257	# The key of the file to translate (ensure it is lowercase)
	258	my $translation_file_key = lc(shift(@_));
	259
	260	# Check that the necessary arguments were supplied
	261	if (!$target_language_code \|\| !$translation_file_key) {
	262	&throw_fatal_error("Missing command argument.");
	263	}
	264
	265	# Get (and check) the translation configuration
	266	my ($source_file, $target_file, $translation_file_type)
	267	= &get_translation_configuration($target_language_code, $translation_file_key);
	268
	269	# Parse the source language and target language files
	270	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	271	my @source_file_lines = &read_file_lines($source_file_path);
	272	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	273
	274	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	275	my @target_file_lines = &read_file_lines($target_file_path);
	276	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	277
	278	# Filter out any automatically translated chunks
	279	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	280	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	281	delete $source_file_key_to_line_mapping{$chunk_key};
	282	delete $target_file_key_to_line_mapping{$chunk_key};
	283	}
	284	}
	285
	286	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	287	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	288	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	289	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	290
	291	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	292	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	293
	294
	295	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	296	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	297	my @uptodate_target_file_keys = ();
	298	foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
	299	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
	300	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
	301
	302	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	303
	304	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	305	# &log_message("Chunk with key $chunk_key needs updating.");
	306	push(@uptodate_target_file_keys, $chunk_key);
	307	}
	308	}
	309
	310	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	311
	312	return $xml_response;
	313	}
	314
	315
[10019]	316	sub get_first_n_chunks_requiring_work
	317	{
	318	# The code of the target language (ensure it is lowercase)
	319	my $target_language_code = lc(shift(@_));
	320	# The key of the file to translate (ensure it is lowercase)
	321	my $translation_file_key = lc(shift(@_));
	322	# The number of chunks to return (defaults to one if not specified)
	323	my $num_chunks_to_return = shift(@_) \|\| "1";
[24627]	324
[10019]	325	# Check that the necessary arguments were supplied
	326	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	327	&throw_fatal_error("Missing command argument.");
[10019]	328	}
[24627]	329
[10019]	330	# Get (and check) the translation configuration
	331	my ($source_file, $target_file, $translation_file_type)
	332	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	333
[10019]	334	# Parse the source language and target language files
	335	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	336	my @source_file_lines = &read_file_lines($source_file_path);
	337	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
[14258]	338
[10019]	339	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	340	my @target_file_lines = &read_file_lines($target_file_path);
	341	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	342
[10019]	343	# Filter out any automatically translated chunks
	344	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	345	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	346	delete $source_file_key_to_line_mapping{$chunk_key};
	347	delete $target_file_key_to_line_mapping{$chunk_key};
	348	}
[10019]	349	}
[24627]	350
[10019]	351	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	352	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	353	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	354	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	355
[10019]	356	# Determine the target file chunks requiring translation
	357	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	358	&log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
[24627]	359
[10019]	360	# Determine the target file chunks requiring updating
	361	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	362	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	363	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	364	&log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
[24627]	365
[18460]	366	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	367
[10040]	368	return $xml_response;
[10019]	369	}
	370
	371
	372	sub get_language_status
	373	{
	374	# The code of the target language (ensure it is lowercase)
	375	my $target_language_code = lc(shift(@_));
[24627]	376
[10019]	377	# Check that the necessary arguments were supplied
	378	if (!$target_language_code) {
[24627]	379	&throw_fatal_error("Missing command argument.");
[10019]	380	}
[24627]	381
[10040]	382	# Form an XML response to the command
	383	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	384	$xml_response .= "<GTIResponse>\n";
	385	$xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
[24627]	386
[18460]	387	foreach my $translation_file (@$gti_translation_files) {
[24627]	388	my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
	389	my $target_file_name = "";
	390
	391	if ($translation_file->{'key'} eq "gs3interface") {
	392	my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
	393	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
	394
	395	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	396	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	397
	398	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	399	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	400	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	401	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
[18460]	402	}
[24627]	403	else {
	404	# Get (and check) the translation configuration
	405	my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
	406	$target_file_name = $target_file;
	407
	408	# Parse the source language and target language files
	409	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	410	my @source_file_lines = &read_file_lines($source_file_path);
	411	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	412
	413	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	414	my @target_file_lines = &read_file_lines($target_file_path);
	415	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	416
	417	# Filter out any automatically translated chunks
	418	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	419	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	420	delete $source_file_key_to_line_mapping{$chunk_key};
	421	delete $target_file_key_to_line_mapping{$chunk_key};
	422	}
	423	}
	424
	425	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	426	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	427
	428	# Determine the target file chunks requiring translation
	429	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	430
	431	# Determine the target file chunks requiring updating
	432	my @target_file_keys_requiring_updating = ();
	433	if (-e $target_file_path) {
	434	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	435	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	436	@target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	437	}
	438
	439	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	440	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	441	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	442	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
	443	}
	444
	445	&log_message("Status of " . $translation_file->{'key'});
	446	&log_message("Number of source chunks: " . $num_source_chunks);
	447	&log_message("Number of target chunks: " . $num_target_chunks);
	448	&log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
	449	&log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
	450
	451	$xml_response .= " <TranslationFile"
[10019]	452	. " key=\"" . $translation_file->{'key'} . "\""
[18460]	453	. " target_file_path=\"" . $target_file_name . "\""
	454	. " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
	455	. " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
	456	. " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
[10019]	457	}
[24627]	458
[10040]	459	$xml_response .= " </LanguageStatus>\n";
[24627]	460
[10040]	461	$xml_response .= "</GTIResponse>\n";
	462	return $xml_response;
[10019]	463	}
	464
	465
	466	sub search_chunks
	467	{
	468	# The code of the target language (ensure it is lowercase)
	469	my $target_language_code = lc(shift(@_));
	470	# The key of the file to translate (ensure it is lowercase)
	471	my $translation_file_key = lc(shift(@_));
	472	# The query string
[10020]	473	my $query_string = join(' ', @_);
[24627]	474
[10019]	475	# Check that the necessary arguments were supplied
	476	if (!$target_language_code \|\| !$translation_file_key \|\| !$query_string) {
[24627]	477	&throw_fatal_error("Missing command argument.");
[10019]	478	}
[24627]	479
[18460]	480	my ($source_file, $target_file, $translation_file_type) = ();
	481	my %source_file_key_to_text_mapping = ();
	482	my %target_file_key_to_text_mapping = ();
	483
	484
	485	if ($translation_file_key ne "gs3interface") {
[24627]	486	# Get (and check) the translation configuration
	487	($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	488
	489	# Parse the source language and target language files
	490	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	491	my @source_file_lines = &read_file_lines($source_file_path);
	492	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	493
	494	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	495	my @target_file_lines = &read_file_lines($target_file_path);
	496	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	497
	498	# Filter out any automatically translated chunks
	499	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	500	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	501	delete $source_file_key_to_line_mapping{$chunk_key};
	502	delete $target_file_key_to_line_mapping{$chunk_key};
	503	}
	504	}
	505
	506	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	507	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[10019]	508	}
[18460]	509	else {
[24627]	510	# Not needed in this case
	511	my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
	512	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	513	\%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
[18460]	514	}
[24627]	515
[10019]	516	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	517	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	518
[10019]	519	# Determine the target file chunks matching the query
	520	my @target_file_keys_matching_query = ();
	521	foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
[24627]	522	my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
	523	if ($target_file_text =~ /$query_string/i) {
	524	# &log_message("Chunk with key $chunk_key matches query.");
	525	push(@target_file_keys_matching_query, $chunk_key);
	526	}
[10019]	527	}
[24627]	528
[10040]	529	# Form an XML response to the command
	530	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	531	$xml_response .= "<GTIResponse>\n";
[24627]	532
[10040]	533	$xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
[10019]	534	foreach my $chunk_key (@target_file_keys_matching_query) {
[24627]	535	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
	536
	537	$xml_response .= " <Chunk key=\"$chunk_key\">\n";
	538	$xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
	539	$xml_response .= " </Chunk>\n";
[10019]	540	}
[10040]	541	$xml_response .= " </ChunksMatchingQuery>\n";
[24627]	542
[10040]	543	$xml_response .= "</GTIResponse>\n";
	544	return $xml_response;
[10019]	545	}
	546
	547
[10050]	548	sub submit_translations
[10019]	549	{
	550	# The code of the target language (ensure it is lowercase)
	551	my $target_language_code = lc(shift(@_));
	552	# The key of the file to translate (ensure it is lowercase)
	553	my $translation_file_key = lc(shift(@_));
[12484]	554	# The username of the translation submitter
	555	my $submitter_username = shift(@_);
[10027]	556	# Whether to submit a target chunk even if it hasn't changed
	557	my $force_submission_flag = shift(@_);
[24627]	558
[10019]	559	# Check that the necessary arguments were supplied
[12484]	560	if (!$target_language_code \|\| !$translation_file_key \|\| !$submitter_username) {
[24627]	561	&log_message("Fatal error (but cannot be thrown): Missing command argument.");
	562	die "\n";
[10019]	563	}
[18460]	564
	565	my %source_file_key_to_text_mapping = ();
	566	my %source_file_key_to_gti_comment_mapping = ();
	567	my %target_file_key_to_text_mapping = ();
	568	my %target_file_key_to_gti_comment_mapping = ();
[24627]	569
[18460]	570	my (@source_file_lines, @target_file_lines) = ();
	571	my ($source_file, $target_file, $translation_file_type);
[24627]	572
[18460]	573
	574	if ($translation_file_key ne "gs3interface") {
[24627]	575	# Get (and check) the translation configuration
	576	($source_file, $target_file, $translation_file_type)
[18460]	577	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	578
	579	# Parse the source language and target language files
	580	@source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	581	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	582	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	583	%source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	584
	585	@target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	586	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	587	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	588	%target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[18460]	589	}
	590	else {
[24627]	591	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	592	\%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
[18460]	593	}
[10019]	594	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	595	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	596
[10019]	597	# Submission date
	598	my $day = (localtime)[3];
	599	my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
	600	my $year = (localtime)[5] + 1900;
	601	my $submission_date = "$day-$month-$year";
[24627]	602
[10019]	603	open(SUBMISSION, "-");
	604	my @submission_lines = <SUBMISSION>;
	605	close(SUBMISSION);
[24627]	606
[10019]	607	# Remove any nasty carriage returns
[18460]	608	# &log_message("Submission:");
[10019]	609	foreach my $submission_line (@submission_lines) {
[24627]	610	$submission_line =~ s/\r$//;
	611	#&log_message(" $submission_line");
[10019]	612	}
[24627]	613
[10019]	614	my %source_file_key_to_submission_mapping = ();
	615	my %target_file_key_to_submission_mapping = ();
	616	for (my $i = 0; $i < scalar(@submission_lines); $i++) {
[24627]	617	# Read source file part of submission
	618	if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
	619	my $chunk_key = $1;
	620
	621	# Read the source file text
	622	my $source_file_chunk_text = "";
	623	$i++;
	624	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
	625	$source_file_chunk_text .= $submission_lines[$i];
	626	$i++;
	627	}
	628	$source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	629	$source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
	630
	631	#&log_message("Source file key: $chunk_key");
	632	#&log_message("Source file text: $source_file_chunk_text");
	633	$source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
	634	}
	635
	636	# Read target file part of submission
	637	if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
	638	my $chunk_key = $1;
	639
	640	# Read the target file text
	641	my $target_file_chunk_text = "";
	642	$i++;
	643	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
	644	$target_file_chunk_text .= $submission_lines[$i];
	645	$i++;
	646	}
	647	$target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	648	$target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
	649
	650	#&log_message("Target file key: $chunk_key");
	651	#&log_message("Target file text: $target_file_chunk_text");
	652	$target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
	653	}
[10019]	654	}
[24627]	655
[10019]	656	# -----------------------------------------
	657	# Validate the translation submissions
	658	# -----------------------------------------
[24627]	659
[10019]	660	# Check that the translations are valid
	661	foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
[24627]	662	# Make sure the submitted chunk still exists in the source file
	663	if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
	664	&log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
	665	delete $source_file_key_to_submission_mapping{$chunk_key};
	666	delete $target_file_key_to_submission_mapping{$chunk_key};
	667	next;
	668	}
	669
	670	# Make sure the submitted source chunk matches the source file chunk
[28518]	671	if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
	672	#if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
[28503]	673	#print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
	674	#print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
	675
[24627]	676	&log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
	677	&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
	678	&log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
	679	delete $source_file_key_to_submission_mapping{$chunk_key};
	680	delete $target_file_key_to_submission_mapping{$chunk_key};
	681	next;
	682	}
[10027]	683	}
[24627]	684
[10027]	685	# Apply the submitted translations
	686	foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
[24627]	687	# Only apply the submission if it is a change, unless -force_submission has been specified
	688	if ($force_submission_flag \|\| !defined($target_file_key_to_text_mapping{$chunk_key}) \|\| $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
	689	$target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
	690	$target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
	691	}
[10019]	692	}
[18460]	693
	694	if ($translation_file_key ne "gs3interface") {
[24627]	695	eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	696	} else {
[24627]	697	eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	698	}
	699	}
[10019]	700
[18460]	701
	702	sub create_glihelp_zip_file
	703	{
	704	my $target_language_code = shift(@_);
	705	my $translation_file_key = "glihelp";
	706
	707	&log_message("Creating GLI Help zip file for $target_language_code");
[24627]	708
[18460]	709	my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
	710
	711	my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
	712	if ( ! -e $classpath) {
[24627]	713	&throw_fatal_error("$classpath doesn't exist! Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
[18460]	714	}
	715
	716	my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
	717	$gli_help_directory = &util::filename_cat($gli_help_directory, "help");
	718
	719	my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
	720	if ( ! -e $gen_many_html_xsl_filepath) {
[24627]	721	&throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
[18460]	722	}
[24627]	723
[18460]	724	my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
	725	my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
	726
	727	my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
	728	$target_file_directory = $target_file_directory."/";
[24627]	729
[18460]	730	my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
[24627]	731
	732	my $perl_exec = &util::get_perl_exec();
	733	my $java_exec = "java";
	734	if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
	735	$java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
	736	}
	737
[24125]	738	my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
[18460]	739	my $response = `$cmd`;
[24627]	740
[24093]	741	$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
[18460]	742	$response = `$cmd`;
[24627]	743
[18460]	744	my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
	745	$cmd = "zip -rj $zip_file_path $target_file_directory -i \.htm \.xml";
	746	$response = `$cmd`;
[10019]	747	}
	748
	749
	750	sub get_translation_configuration
	751	{
	752	# Get the code of the target language
	753	my $target_language_code = shift(@_);
	754	# Get the key of the file to translate
	755	my $translation_file_key = shift(@_);
[24627]	756
[10019]	757	# Read the translation data from the gti.cfg file
	758	my ($source_file, $target_file, $translation_file_type) =
	759	&get_translation_data_for($target_language_code, $translation_file_key);
[24627]	760
[10019]	761	# Check that the file to translate is defined in the gti.cfg file
	762	if (!$source_file \|\| !$target_file \|\| !$translation_file_type) {
[24627]	763	&throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
[10019]	764	}
[24627]	765
[10019]	766	# Check that the source file exists
	767	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	768	if (!-e $source_file_path) {
[24627]	769	&throw_fatal_error("Source file $source_file_path does not exist.");
[10019]	770	}
[24627]	771
[10019]	772	# Check that the source file is up to date
[11104]	773	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[10019]	774	# unless ($translation_file_is_not_in_cvs) {
[14258]	775	#my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
[24627]	776	my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
[10019]	777	if ($source_file_cvs_status =~ /^C /) {
	778	&throw_fatal_error("Source file $source_file_path conflicts with the repository.");
	779	}
	780	if ($source_file_cvs_status =~ /^M /) {
	781	&throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
	782	}
	783	# }
[24627]	784
[10019]	785	return ($source_file, $target_file, $translation_file_type);
	786	}
	787
	788
	789	sub get_translation_data_for
	790	{
	791	my ($target_language_code, $translation_file_key) = @_;
[24627]	792
[10019]	793	foreach my $translation_file (@$gti_translation_files) {
[24627]	794	# If this isn't the correct translation file, move onto the next one
	795	next if ($translation_file_key ne $translation_file->{'key'});
	796
	797	# Resolve the target language file
	798	my $target_language_file = $translation_file->{'target_file'};
	799	if ($target_language_file =~ /(\{.+\;.+\})/) {
	800	my $unresolved_target_language_file_part = $1;
	801
	802	# Check for a special case for the target language code
	803	if ($unresolved_target_language_file_part =~ /(\{\|\;)$target_language_code:([^\;]+)(\;\|\})/) {
	804	my $resolved_target_language_file_part = $2;
	805	$target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
	806	}
	807	# Otherwise use the last part as the default value
	808	else {
	809	my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
	810	$target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
[10019]	811	}
[10041]	812	}
[24627]	813
[10041]	814	# Resolve instances of {iso_639_1_target_language_name}
	815	my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
	816	$iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
	817	$target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
[24627]	818
[10041]	819	# Resolve instances of {target_language_code}
	820	$target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
[24627]	821
[10041]	822	return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
[24627]	823	}
[10019]	824
[24627]	825	return ();
[10019]	826	}
	827
	828
	829	sub read_file_lines
	830	{
	831	my ($file_path) = @_;
[24627]	832
[10019]	833	if (!open(FILE_IN, "<$file_path")) {
[24627]	834	&log_message("Note: Could not open file $file_path.");
	835	return ();
[10019]	836	}
	837	my @file_lines = <FILE_IN>;
	838	close(FILE_IN);
[24627]	839
[10019]	840	return @file_lines;
	841	}
	842
	843
	844	sub build_key_to_line_mapping
	845	{
	846	my ($file_lines, $translation_file_type) = @_;
	847	eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
	848	}
	849
	850
	851	sub build_key_to_text_mapping
	852	{
	853	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	854
[10019]	855	my %key_to_text_mapping = ();
	856	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	857	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	858	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	859
	860	my $chunk_text = @$file_lines[$chunk_starting_line];
	861	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	862	$chunk_text .= @$file_lines[$l];
	863	}
	864
	865	# Map from chunk key to text
	866	eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
[10019]	867	}
[24627]	868
[10019]	869	return %key_to_text_mapping;
	870	}
	871
	872
	873	sub build_key_to_last_update_date_mapping
	874	{
	875	my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	876
[10019]	877	# If the files aren't in CVS then we can't tell anything about what needs updating
	878	# return () if ($translation_file_is_not_in_cvs);
[24627]	879
[10019]	880	# Build a mapping from key to CVS date
	881	# Need to be careful with this mapping because the chunk keys won't necessarily all be valid
	882	my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
[24627]	883
[10019]	884	# Build a mapping from key to comment date
[12483]	885	my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
[24627]	886
[10019]	887	# Build a mapping from key to last update date (the latter of the CVS date and comment date)
	888	my %key_to_last_update_date_mapping = ();
	889	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	890	# Use the CVS date as a starting point
	891	my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
	892	$key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
	893
	894	# If a comment date exists and it is after the CVS date, use that instead
[14258]	895	# need to convert the comment date format to SVN format
[24627]	896	my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
	897	if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
	898	my $chunk_comment_date = $1;
	899	if ((!defined($chunk_cvs_date) \|\| &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
	900	$key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
	901	}
	902	}
[10019]	903	}
[24627]	904
[10019]	905	return %key_to_last_update_date_mapping;
	906	}
	907
	908
	909	sub build_key_to_cvs_date_mapping
	910	{
	911	my ($filename, $translation_file_type) = @_;
[24627]	912
	913	# Use SVN to annotate each line of the file with the date it was last edited
[10019]	914	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[24627]	915	my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
[14258]	916
[10019]	917	my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
[24627]	918
[10019]	919	my @cvs_annotated_file_lines_date = ();
	920	foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
[24627]	921	# Extract the date from the SVN annotation at the front
	922	# svn format : 2007-07-16
[14258]	923	$cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
	924
	925	push(@cvs_annotated_file_lines_date, $1);
	926
	927	# trim extra date information in svn annotation format
	928	# 15:42:49 +1200 (Wed, 21 Jun 2006)
	929	$cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
	930	}
	931
[10019]	932	# Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
	933	my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
[24627]	934
[10019]	935	my %key_to_cvs_date_mapping = ();
	936	foreach my $chunk_key (keys(%key_to_line_mapping)) {
[24627]	937	my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
	938	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
	939
	940	# Find the date this chunk was last edited, from the CVS annotation
	941	my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
	942	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	943	if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
	944	# This part of the chunk has been updated more recently
	945	$chunk_date = $cvs_annotated_file_lines_date[$l];
	946
	947	}
	948	}
	949
	950	# Map from chunk key to CVS date
	951	$key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
[10019]	952	}
[24627]	953
[10019]	954	return %key_to_cvs_date_mapping;
	955	}
	956
	957
[12483]	958	sub build_key_to_gti_comment_mapping
[10019]	959	{
	960	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	961
[12483]	962	my %key_to_gti_comment_mapping = ();
[10019]	963	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	964	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	965	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	966
	967	my $chunk_text = @$file_lines[$chunk_starting_line];
	968	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	969	$chunk_text .= @$file_lines[$l];
	970	}
	971
	972	# Map from chunk key to GTI comment
	973	my $chunk_gti_comment;
	974	eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
	975	$key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
[10019]	976	}
[24627]	977
[12483]	978	return %key_to_gti_comment_mapping;
[10019]	979	}
	980
	981
	982	sub determine_chunks_requiring_translation
	983	{
	984	my $source_file_key_to_text_mapping = shift(@_);
	985	my $target_file_key_to_text_mapping = shift(@_);
[24627]	986
[10019]	987	# Chunks needing translation are those in the source file with no translation in the target file
	988	my @target_file_keys_requiring_translation = ();
	989	foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
[24627]	990	if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
	991	# &log_message("Chunk with key $chunk_key needs translating.");
	992	push(@target_file_keys_requiring_translation, $chunk_key);
	993	}
[10019]	994	}
[24627]	995
[10019]	996	return @target_file_keys_requiring_translation;
	997	}
	998
	999
	1000	sub determine_chunks_requiring_updating
	1001	{
	1002	my $source_file_key_to_last_update_date_mapping = shift(@_);
	1003	my $target_file_key_to_last_update_date_mapping = shift(@_);
[24627]	1004
[10019]	1005	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1006	my @target_file_keys_requiring_updating = ();
	1007	foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
[24627]	1008	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
	1009	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
[14258]	1010
	1011	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
[24627]	1012
[14258]	1013	if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
[24627]	1014	# &log_message("Chunk with key $chunk_key needs updating.");
	1015	push(@target_file_keys_requiring_updating, $chunk_key);
	1016	}
[10019]	1017	}
[24627]	1018
[10019]	1019	return @target_file_keys_requiring_updating;
	1020	}
	1021
	1022
	1023	sub is_chunk_automatically_translated
	1024	{
	1025	my ($chunk_key, $translation_file_type) = @_;
	1026	eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
	1027	}
	1028
	1029
	1030	sub make_text_xml_safe
	1031	{
	1032	my $text = shift(@_);
[11452]	1033	$text =~ s/\&/\&amp\;/g;
	1034	$text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
	1035	$text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
[11498]	1036	$text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
	1037	$text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
[10019]	1038	$text =~ s/</\&lt\;/g;
	1039	$text =~ s/>/\&gt\;/g;
	1040	return $text;
	1041	}
	1042
	1043
[11448]	1044	sub unmake_text_xml_safe
	1045	{
	1046	my $text = shift(@_);
	1047	$text =~ s/\&lt\;/</g;
	1048	$text =~ s/\&gt\;/>/g;
	1049	$text =~ s/\&amp\;/\&/g;
	1050	return $text;
	1051	}
	1052
	1053
[10019]	1054	# Returns 1 if $date1 is after $date2, 0 otherwise
[14258]	1055	sub is_date_after_cvs
[10019]	1056	{
	1057	my ($date1, $date2) = @_;
	1058	my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
[24627]	1059	"Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
	1060
[14258]	1061	if(!defined $date1) {
	1062	return 1;
	1063	}
[24627]	1064
[10019]	1065	my @date1parts = split(/-/, $date1);
	1066	my @date2parts = split(/-/, $date2);
[24627]	1067
[10019]	1068	# Compare year - nasty because we have rolled over into a new century
	1069	my $year1 = $date1parts[2];
	1070	if ($year1 < 80) {
[14258]	1071	$year1 += 2000;
[10019]	1072	}
	1073	my $year2 = $date2parts[2];
	1074	if ($year2 < 80) {
[14258]	1075	$year2 += 2000;
[10019]	1076	}
[24627]	1077
[10019]	1078	# Compare year
	1079	if ($year1 > $year2) {
[24627]	1080	return 1;
[10019]	1081	}
	1082	elsif ($year1 == $year2) {
[24627]	1083	# Year is the same, so compare month
	1084	if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
	1085	return 1;
	1086	}
	1087	elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
	1088	# Month is the same, so compare day
	1089	if ($date1parts[0] > $date2parts[0]) {
	1090	return 1;
	1091	}
	1092	}
[10019]	1093	}
[24627]	1094
[10019]	1095	return 0;
	1096	}
	1097
[14258]	1098	sub is_date_after
	1099	{
	1100	my ($date1, $date2) = @_;
	1101
	1102	if(!defined $date1) {
[24627]	1103	return 1;
[14258]	1104	}
	1105	if(!defined $date2) {
[24627]	1106	return 0;
[14258]	1107	}
	1108
	1109	# 16-Aug-2006
	1110	if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1111	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1112	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1113	$date1=$3 . "-" . $months{$2} . "-" . $1;
	1114	# print "** converted date1: $date1\n";
[14258]	1115	}
	1116	if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1117	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1118	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1119	$date2=$3 . "-" . $months{$2} . "-" . $1;
	1120	# print "** converted date2: $date2\n";
[14258]	1121	}
	1122
	1123
	1124	# 2006-08-16
	1125	my @date1parts = split(/-/, $date1);
	1126	my @date2parts = split(/-/, $date2);
	1127
	1128	# Compare year
	1129	if ($date1parts[0] > $date2parts[0]) {
[24627]	1130	return 1;
[14258]	1131	}
	1132	elsif ($date1parts[0] == $date2parts[0]) {
[24627]	1133	# Year is the same, so compare month
	1134	if ($date1parts[1] > $date2parts[1]) {
	1135	return 1;
	1136	}
	1137	elsif ($date1parts[1] == $date2parts[1]) {
	1138	# Month is the same, so compare day
	1139	if ($date1parts[2] > $date2parts[2]) {
	1140	return 1;
	1141	}
	1142	}
[14258]	1143	}
	1144
	1145	return 0;
	1146	}
[10019]	1147
[14258]	1148
[18460]	1149	sub create_xml_response_for_chunks_requiring_work
	1150	{
	1151	my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
[24627]	1152
[18460]	1153	# Form an XML response to the command
	1154	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1155	$xml_response .= "<GTIResponse>\n";
	1156	$xml_response .= " <TranslationFile"
	1157	. " key=\"" . $translation_file_key . "\""
	1158	. " target_file_path=\"" . $target_file . "\""
	1159	. " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
	1160	. " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
	1161	. " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
[24627]	1162
[18460]	1163	# Do chunks requiring translation first
	1164	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
[24627]	1165	$xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
[18460]	1166	}
	1167	else {
[24627]	1168	$xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1169	}
[24627]	1170
[18460]	1171	my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
	1172	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1173	last if ($num_chunks_to_return == 0);
	1174
	1175	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1176	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1177
	1178	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1179	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1180	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1181	$xml_response .= " </Chunk>\n";
	1182
	1183	$num_chunks_to_return--;
	1184	}
[18460]	1185
	1186	$xml_response .= " </ChunksRequiringTranslation>\n";
[24627]	1187
[18460]	1188	# Then do chunks requiring updating
	1189	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
[24627]	1190	$xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
[18460]	1191	}
	1192	else {
[24627]	1193	$xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1194	}
[24627]	1195
[18460]	1196	# foreach my $chunk_key (@target_file_keys_requiring_updating) {
	1197	@sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
	1198	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1199	last if ($num_chunks_to_return == 0);
	1200
	1201	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1202	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1203	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1204	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1205
	1206	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1207	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1208	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1209	$xml_response .= " </Chunk>\n";
	1210
	1211	$num_chunks_to_return--;
[18460]	1212	}
[24627]	1213
[18460]	1214	$xml_response .= " </ChunksRequiringUpdating>\n";
[24627]	1215
[18460]	1216	$xml_response .= "</GTIResponse>\n";
[24627]	1217
[18460]	1218	return $xml_response;
	1219	}
	1220
[25249]	1221	sub create_xml_response_for_uptodate_chunks
	1222	{
	1223	my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
	1224
	1225	# Form an XML response to the command
	1226	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1227	$xml_response .= "<GTIResponse>\n";
	1228	$xml_response .= " <TranslationFile"
	1229	. " key=\"" . $translation_file_key . "\""
	1230	. " target_file_path=\"" . $target_file . "\""
	1231	. " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
	1232
	1233
	1234	# Then do chunks requiring updating
	1235	$xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
	1236
	1237
	1238	# foreach my $chunk_key (@uptodate_target_file_keys) {
	1239	my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
	1240	foreach my $chunk_key (@sorted_chunk_keys) {
	1241
	1242	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1243	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1244	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1245	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1246
	1247	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1248	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1249	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1250	$xml_response .= " </Chunk>\n";
[18460]	1251
[25249]	1252	}
	1253
	1254	$xml_response .= " </UptodateChunks>\n";
	1255
	1256	$xml_response .= "</GTIResponse>\n";
	1257
	1258	return $xml_response;
	1259	}
	1260
[18460]	1261	sub create_xml_response_for_all_chunks
	1262	{
	1263	my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
[24627]	1264
[18460]	1265	# Form an XML response to the command
	1266	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1267	$xml_response .= "<GTIResponse>\n";
	1268	$xml_response .= " <TranslationFile"
	1269	. " key=\"" . $translation_file_key . "\""
	1270	. " target_file_path=\"" . $target_file . "\"\/>\n";
	1271
	1272	# Do all the chunks
	1273	$xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
[24627]	1274
[18460]	1275	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
	1276	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1277	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1278	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
	1279
	1280	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1281	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1282	if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
	1283	my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1284	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
	1285	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1286	}
	1287	else {
	1288	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1289	}
	1290
	1291	$xml_response .= " </Chunk>\n";
[18460]	1292	}
	1293	$xml_response .= " </Chunks>\n";
	1294
	1295	$xml_response .= "</GTIResponse>\n";
	1296	return $xml_response;
	1297	}
	1298
	1299
	1300
[10019]	1301	# ==========================================================================================
	1302	# MACROFILE FUNCTIONS
	1303
	1304	sub build_key_to_line_mapping_for_macrofile
	1305	{
	1306	my (@file_lines) = @_;
[24627]	1307
[10019]	1308	my $macro_package;
	1309	my %chunk_key_to_line_mapping = ();
	1310	# Process the contents of the file, line by line
	1311	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1312	my $line = $file_lines[$i];
	1313	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1314
	1315	# Check if a new package is being defined
	1316	if ($line =~ m/^package\s+(.+)/) {
	1317	$macro_package = $1;
[10041]	1318	}
[24627]	1319
	1320	# Line contains a macro name
	1321	elsif ($line =~ m/^(_\w+_)/) {
	1322	my $macro_key = $1;
	1323	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1324
	1325	# While there is still text of the macro to go...
	1326	my $startline = $i;
	1327	while ($line !~ /\}$/) {
	1328	$i++;
	1329	if ($i == scalar(@file_lines)) {
	1330	&throw_fatal_error("Could not find end of macro $macro_key.");
	1331	}
	1332	$line = $file_lines[$i];
	1333	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1334	}
	1335
[10019]	1336	# The chunk key consists of the package name and the macro key
	1337	my $chunk_key = $macro_package . "." . $macro_key;
	1338	# Map from chunk key to line
	1339	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1340	}
[24627]	1341
[10019]	1342	# Icon: line in format ## "image text" ## image_type ## macro_name ##
	1343	elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
[24627]	1344	# The chunk key consists of package name and macro key
	1345	my $chunk_key = $macro_package . "." . $1;
	1346	# Map from chunk key to line
	1347	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1348	}
	1349	}
[10019]	1350
[24627]	1351	return %chunk_key_to_line_mapping;
[10019]	1352	}
	1353
	1354
	1355	sub import_chunk_from_macrofile
	1356	{
	1357	my ($chunk_text) = @_;
[24627]	1358
[10019]	1359	# Is this an icon macro??
	1360	if ($chunk_text =~ /^\#\# (.*)/) {
[24627]	1361	# Extract image macro text
	1362	$chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
	1363	$chunk_text = $1;
	1364
[10019]	1365	# Remove enclosing quotes
	1366	$chunk_text =~ s/^\"//;
	1367	$chunk_text =~ s/\"$//;
[24627]	1368	}
[10019]	1369
[24627]	1370	# No, so it must be a text macro
	1371	else {
[10019]	1372	# Remove macro key
	1373	$chunk_text =~ s/^_([^_]+)_(\s*)//;
[24627]	1374
[10019]	1375	# Remove language specifier
	1376	$chunk_text =~ s/^\[l=.\](\s)//;
[24627]	1377
[10019]	1378	# Remove braces enclosing text
	1379	$chunk_text =~ s/^{(\s)((.\|\n))}(\s)(\#.+\s)?/$2/;
[24627]	1380	}
[10019]	1381
[24627]	1382	return $chunk_text;
[10019]	1383	}
	1384
	1385
[12483]	1386	sub get_macrofile_chunk_gti_comment
[10019]	1387	{
	1388	my ($chunk_text) = @_;
[24627]	1389
[10019]	1390	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1391	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1392	return $1;
	1393	}
[10019]	1394
[24627]	1395	return undef;
[10019]	1396	}
	1397
	1398
	1399	sub is_macrofile_chunk_automatically_translated
	1400	{
	1401	my ($chunk_key) = @_;
[24627]	1402
[10019]	1403	# The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
	1404	if ($chunk_key =~ /\._(httpicon\|width\|height)/) {
[24627]	1405	return 1;
[10019]	1406	}
[24627]	1407
[10019]	1408	return 0;
	1409	}
	1410
	1411
	1412	# Use the source file to generate a target file that is formatted the same
	1413	sub write_translated_macrofile
	1414	{
	1415	my $source_file = shift(@_); # Not used
[11321]	1416	my @source_file_lines = @{shift(@_)};
[10019]	1417	my $source_file_key_to_text_mapping = shift(@_);
	1418	my $target_file = shift(@_);
[11321]	1419	my @target_file_lines = @{shift(@_)};
[10019]	1420	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1421	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1422	my $target_language_code = shift(@_);
[24627]	1423
[10126]	1424	# Build a mapping from source file line to chunk key
[11321]	1425	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
[10019]	1426	my %source_file_line_to_key_mapping = ();
	1427	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1428	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1429	}
[10126]	1430	my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
	1431	my $source_file_line_number = 0;
[24627]	1432
[10126]	1433	# Build a mapping from target file line to chunk key
[11321]	1434	my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
[10126]	1435	my %target_file_line_to_key_mapping = ();
	1436	foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
[24627]	1437	$target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10126]	1438	}
	1439	my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
[24627]	1440
[10019]	1441	# Write the new target file
	1442	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1443	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1444	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1445	}
[24627]	1446
[10126]	1447	# Use the header from the target file, to keep language and author information
	1448	if (scalar(@target_file_line_keys) > 0) {
[24627]	1449	my $target_file_line_number = 0;
	1450	my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
	1451	while ($target_file_line_number < $target_file_chunk_starting_line_number) {
	1452	my $target_file_line = $target_file_lines[$target_file_line_number];
	1453	last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
	1454	print TARGET_FILE $target_file_line;
	1455	$target_file_line_number++;
	1456	}
	1457
	1458	$source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
[10126]	1459	}
[24627]	1460
[10019]	1461	# Model the new target file on the source file, with the target file translations
[10126]	1462	foreach my $line_key (@source_file_line_keys) {
[24627]	1463	# Fill in the gaps before this chunk starts
	1464	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1465	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1466	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1467	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1468	$source_file_line_number++;
	1469	}
	1470	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1471
	1472	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1473	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1474	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1475
	1476	my $macrofile_key = $chunk_key;
	1477	$macrofile_key =~ s/^(.+?)\.//;
	1478
	1479	# If no translation exists for this chunk, show this, and move on
	1480	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1481	print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
	1482	next;
	1483	}
	1484
	1485	# Grab the source chunk text
	1486	my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
	1487	for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
	1488	$source_file_chunk .= $source_file_lines[$l];
	1489	}
	1490
	1491	# Is this an icon macro??
	1492	if ($source_file_chunk =~ /^\#\# (.*)/) {
	1493	# Escape any newline and question mark characters so the source text is replaced correctly
	1494	$source_file_chunk_text =~ s/\\/\\\\/g;
[10019]	1495	$source_file_chunk_text =~ s/\?/\\\?/g;
[24627]	1496
[10019]	1497	# Build the new target chunk from the source chunk
	1498	my $target_file_chunk = $source_file_chunk;
	1499	$target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
[10050]	1500	$target_file_chunk =~ s/(\s)*$//;
[10019]	1501	print TARGET_FILE "$target_file_chunk";
	1502	}
[24627]	1503
[10019]	1504	# No, it is just a normal text macro
	1505	else {
	1506	print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
	1507	}
[24627]	1508
[12483]	1509	# Add the "updated" comment, if one exists
	1510	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1511	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
[10050]	1512	}
	1513	print TARGET_FILE "\n";
[24627]	1514	}
[10019]	1515
[24627]	1516	close(TARGET_FILE);
[10019]	1517	}
	1518
	1519
	1520	sub sort_by_line
	1521	{
	1522	return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
	1523	}
	1524
	1525
	1526	# ==========================================================================================
	1527	# RESOURCE BUNDLE FUNCTIONS
	1528
	1529	sub build_key_to_line_mapping_for_resource_bundle
	1530	{
	1531	my (@file_lines) = @_;
[24627]	1532
[11321]	1533	my %chunk_key_to_line_mapping = ();
[10019]	1534	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1535	my $line = $file_lines[$i];
	1536	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1537
	1538	# Line contains a dictionary string
	1539	if ($line =~ /^(\S+?)[:\|=](.*)$/) {
	1540	my $chunk_key = $1;
	1541
	1542	# Map from chunk key to line
	1543	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1544	}
[10019]	1545	}
[24627]	1546
[11321]	1547	return %chunk_key_to_line_mapping;
[10019]	1548	}
	1549
	1550
	1551	sub import_chunk_from_resource_bundle
	1552	{
	1553	my ($chunk_text) = @_;
[24627]	1554
[10019]	1555	# Simple: just remove string key
[18460]	1556	$chunk_text =~ s/^(\S+?)[:\|=](\s*)//;
[10019]	1557	$chunk_text =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
[12484]	1558	$chunk_text =~ s/(\s)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.)\s*$//i;
[24627]	1559
[10019]	1560	return $chunk_text;
	1561	}
	1562
	1563
[12483]	1564	sub get_resource_bundle_chunk_gti_comment
[10019]	1565	{
	1566	my ($chunk_text) = @_;
[24627]	1567
[10019]	1568	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1569	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1570	return $1;
	1571	}
[10019]	1572
[24627]	1573	return undef;
[10019]	1574	}
	1575
	1576
	1577	sub is_resource_bundle_chunk_automatically_translated
	1578	{
	1579	# No resource bundle chunks are automatically translated
	1580	return 0;
	1581	}
	1582
	1583
	1584	sub write_translated_resource_bundle
	1585	{
	1586	my $source_file = shift(@_); # Not used
[11321]	1587	my @source_file_lines = @{shift(@_)};
[10019]	1588	my $source_file_key_to_text_mapping = shift(@_);
	1589	my $target_file = shift(@_);
[11321]	1590	my @target_file_lines = @{shift(@_)}; # Not used
[10019]	1591	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1592	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1593	my $target_language_code = shift(@_); # Not used
[24627]	1594
[10019]	1595	# Build a mapping from chunk key to source file line, and from source file line to chunk key
[11321]	1596	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
[10019]	1597	my %source_file_line_to_key_mapping = ();
	1598	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1599	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1600	}
[24627]	1601
[10019]	1602	# Write the new target file
	1603	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1604	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1605	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1606	}
[24627]	1607
[10019]	1608	# Model the new target file on the source file, with the target file translations
[10091]	1609	my $source_file_line_number = 0;
[10019]	1610	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1611	# Fill in the gaps before this chunk starts
	1612	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1613	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1614	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1615	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1616	$source_file_line_number++;
	1617	}
	1618	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1619
	1620	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1621	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1622	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1623
	1624	# If no translation exists for this chunk, show this, and move on
	1625	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1626	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	1627	next;
	1628	}
	1629
	1630	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	1631	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1632	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1633	}
	1634	print TARGET_FILE "\n";
[10019]	1635	}
[24627]	1636
[10019]	1637	close(TARGET_FILE);
	1638	}
	1639
	1640
[11321]	1641	# ==========================================================================================
	1642	# GREENSTONE XML FUNCTIONS
	1643
	1644	sub build_key_to_line_mapping_for_greenstone_xml
	1645	{
	1646	my (@file_lines) = @_;
[24627]	1647
[11321]	1648	my %chunk_key_to_line_mapping = ();
	1649	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1650	my $line = $file_lines[$i];
	1651	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1652
	1653	# Line contains a string to translate
	1654	if ($line =~ /^\s<Text id=\"(.?)\">/) {
	1655	my $chunk_key = $1;
	1656	$line =~ s/\s*$//; # Remove any nasty whitespace
	1657	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1658
	1659	# While there is still text of the string to go...
	1660	my $startline = $i;
	1661	while ($line !~ /<\/Text>$/) {
	1662	$i++;
	1663	if ($i == scalar(@file_lines)) {
	1664	&throw_fatal_error("Could not find end of string $chunk_key.");
	1665	}
	1666	$line = $file_lines[$i];
	1667	$line =~ s/\s*$//; # Remove any nasty whitespace
	1668	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1669	}
	1670
	1671	# Map from chunk key to line
	1672	if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
	1673	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1674	}
	1675	else {
	1676	&throw_fatal_error("Duplicate key $chunk_key.");
	1677	}
[11321]	1678	}
	1679	}
[24627]	1680
[11321]	1681	return %chunk_key_to_line_mapping;
	1682	}
	1683
	1684
	1685	sub import_chunk_from_greenstone_xml
	1686	{
	1687	my ($chunk_text) = @_;
[24627]	1688
[11321]	1689	# Simple: just remove the Text tags
[11447]	1690	$chunk_text =~ s/^\s<Text id=\"(.?)\">(\s*)//;
[12484]	1691	$chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
[11321]	1692	$chunk_text =~ s/<\/Text>$//;
[24627]	1693
[11321]	1694	return $chunk_text;
	1695	}
	1696
	1697
[12483]	1698	sub get_greenstone_xml_chunk_gti_comment
[11321]	1699	{
	1700	my ($chunk_text) = @_;
[24627]	1701
[11321]	1702	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1703	if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
[24627]	1704	return $1;
[11487]	1705	}
[24627]	1706
[11321]	1707	return undef;
	1708	}
	1709
	1710
	1711	sub is_greenstone_xml_chunk_automatically_translated
	1712	{
	1713	# No greenstone XML chunks are automatically translated
	1714	return 0;
	1715	}
	1716
	1717
	1718	sub write_translated_greenstone_xml
	1719	{
	1720	my $source_file = shift(@_); # Not used
	1721	my @source_file_lines = @{shift(@_)};
	1722	my $source_file_key_to_text_mapping = shift(@_);
	1723	my $target_file = shift(@_);
	1724	my @target_file_lines = @{shift(@_)}; # Not used
	1725	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1726	my $target_file_key_to_gti_comment_mapping = shift(@_);
[11321]	1727	my $target_language_code = shift(@_); # Not used
[24627]	1728
[11321]	1729	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	1730	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
	1731	my %source_file_line_to_key_mapping = ();
	1732	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1733	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[11321]	1734	}
[24627]	1735
[11321]	1736	# Write the new target file
	1737	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1738	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1739	&throw_fatal_error("Could not write target file $target_file_path.");
[11321]	1740	}
[24627]	1741
[11321]	1742	# Model the new target file on the source file, with the target file translations
	1743	my $source_file_line_number = 0;
	1744	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1745	# Fill in the gaps before this chunk starts
	1746	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1747	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1748	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1749	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1750	$source_file_line_number++;
	1751	}
	1752	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1753
	1754	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1755	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1756	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1757	$target_file_chunk_text =~ s/(\n)*$//g;
	1758
	1759	# If no translation exists for this chunk, show this, and move on
	1760	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1761	print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
	1762	next;
	1763	}
	1764
	1765	print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
	1766	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1767	my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1768	$chunk_gti_comment =~ s/^Updated //;
	1769	print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
	1770	}
	1771	print TARGET_FILE "\n";
[11321]	1772	}
[24627]	1773
[11321]	1774	# Fill in the end of the file
	1775	while ($source_file_line_number < scalar(@source_file_lines)) {
[24627]	1776	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1777	$source_file_line_number++;
[11321]	1778	}
[24627]	1779
[11321]	1780	close(TARGET_FILE);
	1781	}
	1782
	1783
[18460]	1784	# ==========================================================================================
	1785	# GREENSTONE3 FUNCTIONS
	1786
	1787	sub get_all_chunks_for_gs3
	1788	{
	1789	# The code of the target language (ensure it is lowercase)
	1790	my $target_language_code = lc(shift(@_));
	1791	my $translation_file_key = lc(shift(@_));
[24627]	1792
[18460]	1793	# Check that the necessary arguments were supplied
	1794	if (!$target_language_code) {
[24627]	1795	&throw_fatal_error("Missing command argument.");
[18460]	1796	}
[24627]	1797
[18460]	1798	# Get (and check) the translation configuration
	1799	# my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	1800
	1801	my %source_files_key_to_text_mapping = ();
	1802	my %target_files_key_to_text_mapping = ();
	1803	my %source_files_key_to_last_update_date_mapping = ();
	1804	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1805
[18460]	1806	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[24627]	1807
[18460]	1808	&log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
	1809	&log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
[24627]	1810
[18460]	1811	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1812	return $xml_response;
	1813	}
	1814
	1815
	1816	sub get_first_n_chunks_requiring_work_for_gs3
	1817	{
	1818	# The code of the target language (ensure it is lowercase)
	1819	my $target_language_code = lc(shift(@_));
	1820	# The key of the file to translate (ensure it is lowercase)
	1821	my $translation_file_key = lc(shift(@_));
	1822	# The number of chunks to return (defaults to one if not specified)
	1823	my $num_chunks_to_return = shift(@_) \|\| "1";
	1824
	1825	# Check that the necessary arguments were supplied
	1826	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	1827	&throw_fatal_error("Missing command argument.");
[18460]	1828	}
	1829
	1830	my %source_files_key_to_text_mapping = ();
	1831	my %target_files_key_to_text_mapping = ();
	1832	my %source_files_key_to_last_update_date_mapping = ();
	1833	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1834
[18460]	1835	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
[24627]	1836	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[18460]	1837
	1838	# Determine the target file chunks requiring translation
	1839	my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
	1840	# Determine the target file chunks requiring updating
	1841	my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1842	&log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
	1843	&log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
[24627]	1844
[18460]	1845	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, "", scalar(keys(%source_files_key_to_text_mapping)),
[24627]	1846	\@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
	1847	$num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1848	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1849
[18460]	1850	return $xml_response;
	1851	}
	1852
[25249]	1853	sub get_uptodate_chunks_for_gs3
	1854	{
	1855	# The code of the target language (ensure it is lowercase)
	1856	my $target_language_code = lc(shift(@_));
	1857	# The key of the file to translate (ensure it is lowercase)
	1858	my $translation_file_key = lc(shift(@_));
	1859	# The number of chunks to return (defaults to one if not specified)
	1860	my $num_chunks_to_return = shift(@_) \|\| "1";
	1861
	1862	# Check that the necessary arguments were supplied
	1863	if (!$target_language_code \|\| !$translation_file_key) {
	1864	&throw_fatal_error("Missing command argument.");
	1865	}
	1866
	1867	my %source_files_key_to_text_mapping = ();
	1868	my %target_files_key_to_text_mapping = ();
	1869	my %source_files_key_to_last_update_date_mapping = ();
	1870	my %target_files_key_to_last_update_date_mapping = ();
	1871
	1872	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1873	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1874
[18460]	1875
[25249]	1876	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1877	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	1878	my @uptodate_target_file_keys = ();
	1879	foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
	1880	my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
	1881	my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
	1882
	1883	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	1884
	1885	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	1886	# &log_message("Chunk with key $chunk_key needs updating.");
	1887	push(@uptodate_target_file_keys, $chunk_key);
	1888	}
	1889	}
[18460]	1890
[25249]	1891	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1892
	1893	return $xml_response;
	1894	}
	1895
	1896
[18460]	1897	sub build_gs3_configuration
	1898	{
	1899	my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
	1900	$source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
	1901
[26544]	1902	my $source_file_directory = &util::filename_cat("WEB-INF","classes"); # my $source_file_directory = "greenstone3";
[18460]	1903	my $translation_file_type = "resource_bundle";
[24627]	1904
[18460]	1905	foreach my $interface_file_key (@gs3_interface_files) {
[24627]	1906
	1907	&log_message("Greenstone 3 interface file: " . $interface_file_key);
	1908
	1909	# Parse the source language and target language files
	1910	my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
	1911	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	1912	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	1913	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	1914	my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	1915
	1916	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
	1917	my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	1918	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	1919	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	1920	my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	1921
	1922
	1923	# Filter out any automatically translated chunks
	1924	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	1925	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	1926	delete $source_file_key_to_line_mapping{$chunk_key};
	1927	delete $target_file_key_to_line_mapping{$chunk_key};
	1928	}
	1929	}
	1930
	1931	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	1932	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	1933
	1934	foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
	1935	my $global_chunk_key = "$interface_file_key.$chunk_key";
	1936	$source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
	1937	$source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
	1938
	1939	if (defined $target_file_key_to_text_mapping{$chunk_key}) {
	1940	$target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
	1941	$target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
	1942	}
	1943	}
[18460]	1944	}
	1945	}
	1946
	1947
	1948	sub write_translated_gs3interface
	1949	{
	1950	my $source_file_key_to_text_mapping = shift(@_);
	1951	my $target_file_key_to_text_mapping = shift(@_);
	1952	my $target_file_key_to_gti_comment_mapping = shift(@_);
	1953	my $target_language_code = shift(@_);
	1954
	1955	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
[24627]	1956
[18460]	1957	my %translated_interface_file_keys = ();
	1958	foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
[24627]	1959	$chunk_key =~ /^([^\.]+)?\.(.*)$/;
	1960	if (!defined $translated_interface_file_keys{$1}) {
	1961	&log_message("Updated interface file: " . $1);
	1962	$translated_interface_file_keys{$1}="";
	1963	}
[18460]	1964	}
	1965	&log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
	1966
	1967	my $source_file_directory = "greenstone3";
	1968
	1969	foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
[24627]	1970
	1971	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	1972	my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
	1973	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	1974	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
	1975	my %source_file_line_to_key_mapping = ();
	1976	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	1977	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[18460]	1978	}
[24627]	1979
	1980	# Write the new target file
	1981	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
	1982	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1983	if (!open(TARGET_FILE, ">$target_file_path")) {
	1984	&throw_fatal_error("Could not write target file $target_file_path.");
	1985	}
	1986
	1987	# Model the new target file on the source file, with the target file translations
	1988	my $source_file_line_number = 0;
	1989	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
	1990	# Fill in the gaps before this chunk starts
	1991	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1992	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1993	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1994	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1995	$source_file_line_number++;
	1996	}
	1997	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1998
	1999	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	2000	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2001	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
	2002	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} \|\| "";
	2003
	2004	# If no translation exists for this chunk, show this, and move on
	2005	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	2006	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	2007	next;
	2008	}
	2009
	2010	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	2011	if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
	2012	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
	2013	}
	2014	print TARGET_FILE "\n";
	2015	}
	2016
	2017	close(TARGET_FILE);
[18460]	2018	}
	2019	}
	2020
[10019]	2021	&main(@ARGV);

Note: See TracBrowser for help on using the repository browser.

Download in other formats: