Context Navigation

source: main/trunk/greenstone2/bin/script/gti.pl@ 30490

Last change on this file since 30490 was 30490, checked in by ak19, 8 years ago
Kathy has updated the core gs3 interface files.
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 90.8 KB

Rev	Line
[10019]	1	#!/usr/bin/perl -w
	2
	3	###########################################################################
	4	#
	5	# gti.pl
	6	#
	7	# A component of the Greenstone digital library software
	8	# from the New Zealand Digital Library Project at the
	9	# University of Waikato, New Zealand.
	10	#
	11	# Copyright (C) 2005 New Zealand Digital Library Project
	12	#
	13	# This program is free software; you can redistribute it and/or modify
	14	# it under the terms of the GNU General Public License as published by
	15	# the Free Software Foundation; either version 2 of the License, or
	16	# (at your option) any later version.
	17	#
	18	# This program is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	# GNU General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU General Public License
	24	# along with this program; if not, write to the Free Software
	25	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	26	#
	27	###########################################################################
	28
	29
	30	BEGIN {
	31	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
	32	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
	33	}
	34
[24627]	35
[10019]	36	use iso639;
	37	use strict;
	38	use util;
[29415]	39	use FileUtils;
[10019]	40
	41	my $gsdl_root_directory = "$ENV{'GSDLHOME'}";
	42	my $gti_log_file = &util::filename_cat($gsdl_root_directory, "etc", "gti.log");
[28976]	43	my $source_language_code = "en"; # This is non-negotiable
[10019]	44
	45	my $gti_translation_files =
[24627]	46	[ # Greenstone macrofiles
	47	{ 'key' => "coredm",
[10019]	48	'file_type' => "macrofile",
	49	'source_file' => "macros/english.dm",
[11224]	50	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[24627]	51
	52	{ 'key' => "auxdm",
[10019]	53	'file_type' => "macrofile",
	54	'source_file' => "macros/english2.dm",
[11224]	55	'target_file' => "macros/{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}2.dm" },
[10019]	56
[29411]	57	#{ 'key' => "paperspastdm",
	58	# 'file_type' => "macrofile",
	59	# 'source_file' => "macros/paperspast-english.dm",
	60	# 'target_file' => "macros/paperspast-{bn:bengali;fa:farsi;gd:gaelic;id:indo;lv:latvian;pt-br:port-br;pt-pt:port-pt;zh-tr:chinese-trad;iso_639_1_target_language_name}.dm" },
[25285]	61
[24627]	62	# GLI dictionary
	63	{ 'key' => "glidict",
[13946]	64	'file_type' => "resource_bundle",
	65	'source_file' => "gli/classes/dictionary.properties",
	66	'target_file' => "gli/classes/dictionary_{target_language_code}.properties" },
[10114]	67
[24627]	68	# GLI help
	69	{ 'key' => "glihelp",
[13946]	70	'file_type' => "greenstone_xml",
	71	'source_file' => "gli/help/en/help.xml",
	72	'target_file' => "gli/help/{target_language_code}/help.xml" },
[11602]	73
[24627]	74	# Greenstone Perl modules
	75	{ 'key' => "perlmodules",
[10019]	76	'file_type' => "resource_bundle",
[11634]	77	'source_file' => "perllib/strings.properties",
	78	'target_file' => "perllib/strings_{target_language_code}.properties" },
[10019]	79
[28337]	80	# Greenstone Installer interface
	81	{ 'key' => "gsinstaller",
	82	'file_type' => "resource_bundle",
	83	'source_file' => "gsinstaller/LanguagePack.properties",
	84	'target_file' => "gsinstaller/LanguagePack_{target_language_code}.properties" },
	85
[24627]	86	# Greenstone tutorial exercises
	87	# { 'key' => "tutorials",
	88	# 'file_type' => "greenstone_xml",
	89	# 'source_file' => "gsdl-documentation/tutorials/xml-source/tutorial_en.xml",
	90	# 'target_file' => "gsdl-documentation/tutorials/xml-source/tutorial_{target_language_code}.xml" },
[11366]	91
[24627]	92	# new Greenstone.org
	93	{ 'key' => "greenorg",
	94	'file_type' => "resource_bundle",
	95	'source_file' => "greenstoneorg/website/classes/Gsc.properties",
	96	'target_file' => "greenstoneorg/website/classes/Gsc_{target_language_code}.properties"
[29411]	97	},
	98
[29412]	99	# greenstone 3 interface files, from http://svn.greenstone.org/main/trunk/greenstone3/web/WEB-INF/classes
[29413]	100	# check it out as greenstone3
[29411]	101	{ 'key' => "gs3interface",
[29415]	102	'file_type' => "resource_bundle",
	103	'source_file' => "greenstone3",
	104	'target_file' => "greenstone3"
[24627]	105	}
	106	];
[10019]	107
[30490]	108	my @gs3_interface_files = ("interface_default", "ServiceRack", "metadata_names");
	109	#"AbstractBrowse", "AbstractGS2FieldSearch", "AbstractSearch", "AbstractTextSearch", "Authentication", "CrossCollectionSearch", "GS2LuceneSearch", "LuceneSearch", "MapRetrieve", "MapSearch", "PhindPhraseBrowse", "SharedSoleneGS2FieldSearch");
[11026]	110
[26547]	111	# Auxilliary GS3 interface files. This list is not used at present
	112	# Combine with above list if generating translation spreadsheet for all interface files
[30490]	113	my @gs3_aux_interface_files = ("GATEServices","QBRWebServicesHelp", "Visualizer", "IViaSearch", "GS2Construct");
[18460]	114
[30490]	115	my @gs3_other_interface_files = ("interface_default2", "interface_basic", "interface_basic2", "interface_nzdl", "interface_gs2");
	116
[26547]	117	# Not: i18n, log4j
	118
[10019]	119	sub main
	120	{
	121	# Get the command to process, and any arguments
	122	my $gti_command = shift(@_);
	123	my @gti_command_arguments = @_;
[18460]	124	my $module = $_[1];
[26544]	125
	126	# for GS3, set gsdl_root_dir to GSDL3HOME
[29411]	127	#if($module && $module eq "gs3interface"){ # module is empty when the gti-command is create-glihelp-zip-file
	128	#if($ENV{'GSDL3SRCHOME'}) {
	129	# $gsdl_root_directory = (defined $ENV{'GSDL3HOME'}) ? $ENV{'GSDL3HOME'} : &util::filename_cat($ENV{'GSDL3SRCHOME'}, "web");
	130	# $gti_log_file = &util::filename_cat($gsdl_root_directory, "logs", "gti.log");
	131	#}
	132	#}
[24627]	133
[10019]	134	# Open the GTI log file for appending, or write to STDERR if that fails
	135	if (!open(GTI_LOG, ">>$gti_log_file")) {
[24627]	136	open(GTI_LOG, ">&STDERR");
[10019]	137	}
[24627]	138
[10019]	139	# Log the command that launched this script
	140	&log_message("Command: $0 @ARGV");
[24627]	141
[10019]	142	# Check that a command was supplied
	143	if (!$gti_command) {
[24627]	144	&throw_fatal_error("Missing command.");
[18460]	145	}
[24627]	146
[10019]	147	# Process the command
[13948]	148	if ($gti_command =~ /^get-all-chunks$/i) {
[24627]	149	# Check that GS3 interface is the target
[26544]	150	if ($module eq "gs3interface") {
[24627]	151	print &get_all_chunks_for_gs3(@gti_command_arguments);
	152	} else {
	153	print &get_all_chunks(@gti_command_arguments);
	154	}
[13948]	155	}
[24627]	156	elsif ($gti_command =~ /^get-first-n-chunks-requiring-work$/i) {
	157	if ($module eq "gs3interface") {
	158	print &get_first_n_chunks_requiring_work_for_gs3(@gti_command_arguments);
	159	} else {
	160	print &get_first_n_chunks_requiring_work(@gti_command_arguments);
	161	}
[10019]	162	}
[25249]	163	elsif ($gti_command =~ /^get-uptodate-chunks$/i) {
	164	if ($module eq "gs3interface") {
	165	print &get_uptodate_chunks_for_gs3(@gti_command_arguments);
	166	} else {
	167	print &get_uptodate_chunks(@gti_command_arguments);
	168	}
	169	}
[24627]	170	elsif ($gti_command =~ /^get-language-status$/i) {
	171	print &get_language_status(@gti_command_arguments);
[10019]	172	}
[24627]	173	elsif ($gti_command =~ /^search-chunks$/i) {
	174	print &search_chunks(@gti_command_arguments);
[10019]	175	}
[24627]	176	elsif ($gti_command =~ /^submit-translations$/i) {
	177	# This command cannot produce any output since it reads input
	178	&submit_translations(@gti_command_arguments);
[10019]	179	}
[24627]	180	elsif ($gti_command =~ /^create-glihelp-zip-file$/i) {
	181	# This command cannot produce any output since it reads input
	182	&create_glihelp_zip_file(@gti_command_arguments);
[18460]	183	}
[24627]	184	else {
	185	# The command was not recognized
	186	&throw_fatal_error("Unknown command \"$gti_command\".");
	187	}
[10019]	188	}
	189
	190
	191	sub throw_fatal_error
	192	{
	193	my $error_message = shift(@_);
[24627]	194
[10019]	195	# Write an XML error response
	196	print "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	197	print "<GTIResponse>\n";
	198	print " <GTIError time=\"" . time() . "\">" . $error_message . "</GTIError>\n";
	199	print "</GTIResponse>\n";
[24627]	200
[10019]	201	# Log the error message, then die
	202	&log_message("Error: $error_message");
	203	die "\n";
	204	}
	205
	206
	207	sub log_message
	208	{
	209	my $log_message = shift(@_);
	210	print GTI_LOG time() . " -- " . $log_message . "\n";
	211	}
	212
	213
[13948]	214	sub get_all_chunks
	215	{
	216	# The code of the target language (ensure it is lowercase)
	217	my $target_language_code = lc(shift(@_));
	218	# The key of the file to translate (ensure it is lowercase)
	219	my $translation_file_key = lc(shift(@_));
[24627]	220
[13948]	221	# Check that the necessary arguments were supplied
	222	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	223	&throw_fatal_error("Missing command argument.");
[13948]	224	}
[24627]	225
[13948]	226	# Get (and check) the translation configuration
	227	my ($source_file, $target_file, $translation_file_type)
	228	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	229
[13948]	230	# Parse the source language and target language files
	231	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	232	my @source_file_lines = &read_file_lines($source_file_path);
	233	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	234
	235	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	236	my @target_file_lines = &read_file_lines($target_file_path);
	237	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	238
[13948]	239	# Filter out any automatically translated chunks
	240	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	241	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	242	delete $source_file_key_to_line_mapping{$chunk_key};
	243	delete $target_file_key_to_line_mapping{$chunk_key};
	244	}
[13948]	245	}
[24627]	246
[13948]	247	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	248	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	249	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	250	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	251
[13948]	252	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	253	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[24627]	254
[18460]	255	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, $target_file, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
[13948]	256
	257	return $xml_response;
	258	}
	259
	260
[25249]	261	sub get_uptodate_chunks
	262	{
	263	# The code of the target language (ensure it is lowercase)
	264	my $target_language_code = lc(shift(@_));
	265	# The key of the file to translate (ensure it is lowercase)
	266	my $translation_file_key = lc(shift(@_));
	267
	268	# Check that the necessary arguments were supplied
	269	if (!$target_language_code \|\| !$translation_file_key) {
	270	&throw_fatal_error("Missing command argument.");
	271	}
	272
	273	# Get (and check) the translation configuration
	274	my ($source_file, $target_file, $translation_file_type)
	275	= &get_translation_configuration($target_language_code, $translation_file_key);
	276
	277	# Parse the source language and target language files
	278	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	279	my @source_file_lines = &read_file_lines($source_file_path);
	280	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	281
	282	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	283	my @target_file_lines = &read_file_lines($target_file_path);
	284	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	285
	286	# Filter out any automatically translated chunks
	287	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	288	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	289	delete $source_file_key_to_line_mapping{$chunk_key};
	290	delete $target_file_key_to_line_mapping{$chunk_key};
	291	}
	292	}
	293
	294	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	295	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	296	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	297	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	298
	299	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	300	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	301
	302
	303	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	304	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	305	my @uptodate_target_file_keys = ();
	306	foreach my $chunk_key (keys(%source_file_key_to_last_update_date_mapping)) {
	307	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping{$chunk_key};
	308	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping{$chunk_key};
	309
	310	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	311
	312	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	313	# &log_message("Chunk with key $chunk_key needs updating.");
	314	push(@uptodate_target_file_keys, $chunk_key);
	315	}
	316	}
	317
	318	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, $target_file, \@uptodate_target_file_keys, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	319
	320	return $xml_response;
	321	}
	322
	323
[10019]	324	sub get_first_n_chunks_requiring_work
	325	{
	326	# The code of the target language (ensure it is lowercase)
	327	my $target_language_code = lc(shift(@_));
	328	# The key of the file to translate (ensure it is lowercase)
	329	my $translation_file_key = lc(shift(@_));
	330	# The number of chunks to return (defaults to one if not specified)
	331	my $num_chunks_to_return = shift(@_) \|\| "1";
[24627]	332
[10019]	333	# Check that the necessary arguments were supplied
	334	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	335	&throw_fatal_error("Missing command argument.");
[10019]	336	}
[24627]	337
[10019]	338	# Get (and check) the translation configuration
	339	my ($source_file, $target_file, $translation_file_type)
	340	= &get_translation_configuration($target_language_code, $translation_file_key);
[28755]	341
[10019]	342	# Parse the source language and target language files
	343	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	344	my @source_file_lines = &read_file_lines($source_file_path);
	345	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
[14258]	346
[10019]	347	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	348	my @target_file_lines = &read_file_lines($target_file_path);
	349	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
[24627]	350
[10019]	351	# Filter out any automatically translated chunks
	352	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	353	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	354	delete $source_file_key_to_line_mapping{$chunk_key};
	355	delete $target_file_key_to_line_mapping{$chunk_key};
	356	}
[10019]	357	}
[24627]	358
[10019]	359	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	360	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	361	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	362	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	363
[10019]	364	# Determine the target file chunks requiring translation
	365	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	366	&log_message("Number of target chunks requiring translation: " . scalar(@target_file_keys_requiring_translation));
[24627]	367
[10019]	368	# Determine the target file chunks requiring updating
	369	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	370	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	371	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	372	&log_message("Number of target chunks requiring updating: " . scalar(@target_file_keys_requiring_updating));
[24627]	373
[18460]	374	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $target_file, scalar(keys(%source_file_key_to_text_mapping)), \@target_file_keys_requiring_translation, \@target_file_keys_requiring_updating, $num_chunks_to_return, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	375
[10040]	376	return $xml_response;
[10019]	377	}
	378
	379
	380	sub get_language_status
	381	{
	382	# The code of the target language (ensure it is lowercase)
	383	my $target_language_code = lc(shift(@_));
[24627]	384
[10019]	385	# Check that the necessary arguments were supplied
	386	if (!$target_language_code) {
[24627]	387	&throw_fatal_error("Missing command argument.");
[10019]	388	}
[24627]	389
[10040]	390	# Form an XML response to the command
	391	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	392	$xml_response .= "<GTIResponse>\n";
	393	$xml_response .= " <LanguageStatus code=\"$target_language_code\">\n";
[24627]	394
[18460]	395	foreach my $translation_file (@$gti_translation_files) {
[24627]	396	my ($num_source_chunks, $num_target_chunks, $num_chunks_requiring_translation, $num_chunks_requiring_updating) = 0;
	397	my $target_file_name = "";
	398
	399	if ($translation_file->{'key'} eq "gs3interface") {
	400	my (%source_file_key_to_text_mapping, %target_file_key_to_text_mapping, %source_file_key_to_last_update_date_mapping, %target_file_key_to_last_update_date_mapping ) = ();
	401	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping, \%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping );
	402
	403	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	404	my @target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	405
	406	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	407	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	408	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	409	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
[18460]	410	}
[24627]	411	else {
	412	# Get (and check) the translation configuration
	413	my ($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file->{'key'});
	414	$target_file_name = $target_file;
	415
	416	# Parse the source language and target language files
	417	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	418	my @source_file_lines = &read_file_lines($source_file_path);
	419	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	420
	421	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	422	my @target_file_lines = &read_file_lines($target_file_path);
	423	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	424
	425	# Filter out any automatically translated chunks
	426	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	427	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	428	delete $source_file_key_to_line_mapping{$chunk_key};
	429	delete $target_file_key_to_line_mapping{$chunk_key};
	430	}
	431	}
	432
	433	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	434	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	435
	436	# Determine the target file chunks requiring translation
	437	my @target_file_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping);
	438
	439	# Determine the target file chunks requiring updating
	440	my @target_file_keys_requiring_updating = ();
	441	if (-e $target_file_path) {
	442	my %source_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($source_file, \@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	443	my %target_file_key_to_last_update_date_mapping = &build_key_to_last_update_date_mapping($target_file, \@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	444	@target_file_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_file_key_to_last_update_date_mapping, \%target_file_key_to_last_update_date_mapping);
	445	}
	446
	447	$num_source_chunks = scalar(keys(%source_file_key_to_text_mapping));
	448	$num_target_chunks = scalar(keys(%target_file_key_to_text_mapping));
	449	$num_chunks_requiring_translation = scalar(@target_file_keys_requiring_translation);
	450	$num_chunks_requiring_updating = scalar(@target_file_keys_requiring_updating);
	451	}
	452
	453	&log_message("Status of " . $translation_file->{'key'});
	454	&log_message("Number of source chunks: " . $num_source_chunks);
	455	&log_message("Number of target chunks: " . $num_target_chunks);
	456	&log_message("Number of target chunks requiring translation: " . $num_chunks_requiring_translation);
	457	&log_message("Number of target chunks requiring updating: " . $num_chunks_requiring_updating);
	458
	459	$xml_response .= " <TranslationFile"
[10019]	460	. " key=\"" . $translation_file->{'key'} . "\""
[18460]	461	. " target_file_path=\"" . $target_file_name . "\""
	462	. " num_chunks_translated=\"" . ($num_source_chunks - $num_chunks_requiring_translation) . "\""
	463	. " num_chunks_requiring_translation=\"" . $num_chunks_requiring_translation . "\""
	464	. " num_chunks_requiring_updating=\"" . $num_chunks_requiring_updating . "\"\/>\n";
[10019]	465	}
[24627]	466
[10040]	467	$xml_response .= " </LanguageStatus>\n";
[24627]	468
[10040]	469	$xml_response .= "</GTIResponse>\n";
	470	return $xml_response;
[10019]	471	}
	472
	473
	474	sub search_chunks
	475	{
	476	# The code of the target language (ensure it is lowercase)
	477	my $target_language_code = lc(shift(@_));
	478	# The key of the file to translate (ensure it is lowercase)
	479	my $translation_file_key = lc(shift(@_));
	480	# The query string
[10020]	481	my $query_string = join(' ', @_);
[24627]	482
[10019]	483	# Check that the necessary arguments were supplied
	484	if (!$target_language_code \|\| !$translation_file_key \|\| !$query_string) {
[24627]	485	&throw_fatal_error("Missing command argument.");
[10019]	486	}
[24627]	487
[18460]	488	my ($source_file, $target_file, $translation_file_type) = ();
	489	my %source_file_key_to_text_mapping = ();
	490	my %target_file_key_to_text_mapping = ();
	491
	492
	493	if ($translation_file_key ne "gs3interface") {
[24627]	494	# Get (and check) the translation configuration
	495	($source_file, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	496
	497	# Parse the source language and target language files
	498	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	499	my @source_file_lines = &read_file_lines($source_file_path);
	500	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	501
	502	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	503	my @target_file_lines = &read_file_lines($target_file_path);
	504	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	505
	506	# Filter out any automatically translated chunks
	507	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	508	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	509	delete $source_file_key_to_line_mapping{$chunk_key};
	510	delete $target_file_key_to_line_mapping{$chunk_key};
	511	}
	512	}
	513
	514	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	515	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[10019]	516	}
[18460]	517	else {
[24627]	518	# Not needed in this case
	519	my (%source_file_key_to_gti_command_mapping, %target_file_key_to_gti_command_mapping) = ();
	520	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	521	\%source_file_key_to_gti_command_mapping, \%target_file_key_to_gti_command_mapping);
[18460]	522	}
[24627]	523
[10019]	524	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	525	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	526
[10019]	527	# Determine the target file chunks matching the query
	528	my @target_file_keys_matching_query = ();
	529	foreach my $chunk_key (keys(%target_file_key_to_text_mapping)) {
[24627]	530	my $target_file_text = $target_file_key_to_text_mapping{$chunk_key};
	531	if ($target_file_text =~ /$query_string/i) {
	532	# &log_message("Chunk with key $chunk_key matches query.");
	533	push(@target_file_keys_matching_query, $chunk_key);
	534	}
[10019]	535	}
[24627]	536
[10040]	537	# Form an XML response to the command
	538	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	539	$xml_response .= "<GTIResponse>\n";
[24627]	540
[10040]	541	$xml_response .= " <ChunksMatchingQuery size=\"" . scalar(@target_file_keys_matching_query) . "\">\n";
[10019]	542	foreach my $chunk_key (@target_file_keys_matching_query) {
[24627]	543	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping{$chunk_key});
	544
	545	$xml_response .= " <Chunk key=\"$chunk_key\">\n";
	546	$xml_response .= " <TargetFileText>$target_file_chunk_text</TargetFileText>\n";
	547	$xml_response .= " </Chunk>\n";
[10019]	548	}
[10040]	549	$xml_response .= " </ChunksMatchingQuery>\n";
[24627]	550
[10040]	551	$xml_response .= "</GTIResponse>\n";
	552	return $xml_response;
[10019]	553	}
	554
	555
[10050]	556	sub submit_translations
[10019]	557	{
	558	# The code of the target language (ensure it is lowercase)
	559	my $target_language_code = lc(shift(@_));
	560	# The key of the file to translate (ensure it is lowercase)
	561	my $translation_file_key = lc(shift(@_));
[12484]	562	# The username of the translation submitter
	563	my $submitter_username = shift(@_);
[10027]	564	# Whether to submit a target chunk even if it hasn't changed
	565	my $force_submission_flag = shift(@_);
[24627]	566
[10019]	567	# Check that the necessary arguments were supplied
[12484]	568	if (!$target_language_code \|\| !$translation_file_key \|\| !$submitter_username) {
[24627]	569	&log_message("Fatal error (but cannot be thrown): Missing command argument.");
	570	die "\n";
[10019]	571	}
[18460]	572
	573	my %source_file_key_to_text_mapping = ();
	574	my %source_file_key_to_gti_comment_mapping = ();
	575	my %target_file_key_to_text_mapping = ();
	576	my %target_file_key_to_gti_comment_mapping = ();
[24627]	577
[18460]	578	my (@source_file_lines, @target_file_lines) = ();
	579	my ($source_file, $target_file, $translation_file_type);
[24627]	580
[18460]	581
	582	if ($translation_file_key ne "gs3interface") {
[24627]	583	# Get (and check) the translation configuration
	584	($source_file, $target_file, $translation_file_type)
[18460]	585	= &get_translation_configuration($target_language_code, $translation_file_key);
[24627]	586
	587	# Parse the source language and target language files
	588	@source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	589	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	590	%source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	591	%source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	592
	593	@target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	594	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	595	%target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	596	%target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
[18460]	597	}
	598	else {
[24627]	599	&build_gs3_configuration($target_language_code, \%source_file_key_to_text_mapping, \%target_file_key_to_text_mapping,
	600	\%source_file_key_to_gti_comment_mapping, \%target_file_key_to_gti_comment_mapping);
[18460]	601	}
[10019]	602	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	603	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
[24627]	604
[10019]	605	# Submission date
	606	my $day = (localtime)[3];
	607	my $month = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[(localtime)[4]];
	608	my $year = (localtime)[5] + 1900;
	609	my $submission_date = "$day-$month-$year";
[24627]	610
[10019]	611	open(SUBMISSION, "-");
	612	my @submission_lines = <SUBMISSION>;
	613	close(SUBMISSION);
[24627]	614
[10019]	615	# Remove any nasty carriage returns
[18460]	616	# &log_message("Submission:");
[10019]	617	foreach my $submission_line (@submission_lines) {
[24627]	618	$submission_line =~ s/\r$//;
	619	#&log_message(" $submission_line");
[10019]	620	}
[24627]	621
[10019]	622	my %source_file_key_to_submission_mapping = ();
	623	my %target_file_key_to_submission_mapping = ();
	624	for (my $i = 0; $i < scalar(@submission_lines); $i++) {
[24627]	625	# Read source file part of submission
	626	if ($submission_lines[$i] =~ /^\<SourceFileText key=\"(.+)\"\>/) {
	627	my $chunk_key = $1;
	628
	629	# Read the source file text
	630	my $source_file_chunk_text = "";
	631	$i++;
	632	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/SourceFileText\>/) {
	633	$source_file_chunk_text .= $submission_lines[$i];
	634	$i++;
	635	}
	636	$source_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	637	$source_file_chunk_text = &unmake_text_xml_safe($source_file_chunk_text);
	638
	639	#&log_message("Source file key: $chunk_key");
	640	#&log_message("Source file text: $source_file_chunk_text");
	641	$source_file_key_to_submission_mapping{$chunk_key} = $source_file_chunk_text;
	642	}
	643
	644	# Read target file part of submission
	645	if ($submission_lines[$i] =~ /^\<TargetFileText key=\"(.+)\"\>/) {
	646	my $chunk_key = $1;
	647
	648	# Read the target file text
	649	my $target_file_chunk_text = "";
	650	$i++;
	651	while ($i < scalar(@submission_lines) && $submission_lines[$i] !~ /^\<\/TargetFileText\>/) {
	652	$target_file_chunk_text .= $submission_lines[$i];
	653	$i++;
	654	}
	655	$target_file_chunk_text =~ s/\n$//; # Strip the extra newline character added
	656	$target_file_chunk_text = &unmake_text_xml_safe($target_file_chunk_text);
	657
	658	#&log_message("Target file key: $chunk_key");
	659	#&log_message("Target file text: $target_file_chunk_text");
	660	$target_file_key_to_submission_mapping{$chunk_key} = $target_file_chunk_text;
	661	}
[10019]	662	}
[24627]	663
[10019]	664	# -----------------------------------------
	665	# Validate the translation submissions
	666	# -----------------------------------------
[24627]	667
[10019]	668	# Check that the translations are valid
	669	foreach my $chunk_key (keys(%source_file_key_to_submission_mapping)) {
[24627]	670	# Make sure the submitted chunk still exists in the source file
	671	if (!defined($source_file_key_to_text_mapping{$chunk_key})) {
	672	&log_message("Warning: Source chunk $chunk_key no longer exists (ignoring submission).");
	673	delete $source_file_key_to_submission_mapping{$chunk_key};
	674	delete $target_file_key_to_submission_mapping{$chunk_key};
	675	next;
	676	}
	677
	678	# Make sure the submitted source chunk matches the source file chunk
[28518]	679	if ($source_file_key_to_submission_mapping{$chunk_key} ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
	680	#if (&unmake_text_xml_safe($source_file_key_to_submission_mapping{$chunk_key}) ne &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key})) {
[28503]	681	#print STDERR "**** $source_file_key_to_submission_mapping{$chunk_key}\n";
	682	#print STDERR "**** " . &unmake_text_xml_safe($source_file_key_to_text_mapping{$chunk_key}) ."\n";
	683
[24627]	684	&log_message("Warning: Source chunk $chunk_key has changed (ignoring submission).");
	685	&log_message("Submission source: $source_file_key_to_submission_mapping{$chunk_key}");
	686	&log_message(" Source text: $source_file_key_to_text_mapping{$chunk_key}");
	687	delete $source_file_key_to_submission_mapping{$chunk_key};
	688	delete $target_file_key_to_submission_mapping{$chunk_key};
	689	next;
	690	}
[10027]	691	}
[24627]	692
[10027]	693	# Apply the submitted translations
	694	foreach my $chunk_key (keys(%target_file_key_to_submission_mapping)) {
[24627]	695	# Only apply the submission if it is a change, unless -force_submission has been specified
	696	if ($force_submission_flag \|\| !defined($target_file_key_to_text_mapping{$chunk_key}) \|\| $target_file_key_to_submission_mapping{$chunk_key} ne $target_file_key_to_text_mapping{$chunk_key}) {
	697	$target_file_key_to_text_mapping{$chunk_key} = $target_file_key_to_submission_mapping{$chunk_key};
	698	$target_file_key_to_gti_comment_mapping{$chunk_key} = "Updated $submission_date by $submitter_username";
	699	}
[10019]	700	}
[18460]	701
	702	if ($translation_file_key ne "gs3interface") {
[24627]	703	eval "&write_translated_${translation_file_type}(\$source_file, \\\@source_file_lines, \\\%source_file_key_to_text_mapping, \$target_file, \\\@target_file_lines, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	704	} else {
[24627]	705	eval "&write_translated_gs3interface(\\\%source_file_key_to_text_mapping, \\\%target_file_key_to_text_mapping, \\\%target_file_key_to_gti_comment_mapping, \$target_language_code)";
[18460]	706	}
	707	}
[10019]	708
[18460]	709
	710	sub create_glihelp_zip_file
	711	{
	712	my $target_language_code = shift(@_);
	713	my $translation_file_key = "glihelp";
	714
	715	&log_message("Creating GLI Help zip file for $target_language_code");
[24627]	716
[18460]	717	my ($source_file, $target_file, $translation_file_type) = &get_translation_data_for($target_language_code, $translation_file_key);
	718
	719	my $classpath = &util::filename_cat($gsdl_root_directory, "gti-lib");
[28977]	720	my $oldclasspath = $classpath;
[18460]	721	if ( ! -e $classpath) {
[28977]	722	$classpath = &util::filename_cat($gsdl_root_directory, "gli", "shared");
	723	}
	724	if ( ! -e $classpath) {
	725	&throw_fatal_error("$classpath doesn't exist! (Neither does $oldclasspath.) Need the files in this directory (ApplyXLST and its related files) to create the zip file for GLI Help");
	726	}
	727
[28991]	728
	729	my $perllib_path = &util::filename_cat($gsdl_root_directory, "perllib"); # strings.properties
	730	my $gliclasses_path = &util::filename_cat($gsdl_root_directory, "gli", "classes"); # dictionary.properties
	731	my $os = $^O;
	732	my $path_separator = ($^O =~ m/mswin/i) ? ";" : ":";
	733	my $xalan_path = &util::filename_cat($classpath, "xalan.jar");
	734	$classpath = "$perllib_path$path_separator$gliclasses_path$path_separator$classpath$path_separator$xalan_path";
	735
[18460]	736	my $gli_help_directory = &util::filename_cat($gsdl_root_directory, "gli");
	737	$gli_help_directory = &util::filename_cat($gli_help_directory, "help");
	738
	739	my $gen_many_html_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-many-html.xsl");
	740	if ( ! -e $gen_many_html_xsl_filepath) {
[24627]	741	&throw_fatal_error("$gen_many_html_xsl_filepath doesn't exist! Need this file to create the zip file for GLI Help");
[18460]	742	}
[24627]	743
[18460]	744	my $gen_index_xml_xsl_filepath = &util::filename_cat($gli_help_directory, "gen-index-xml.xsl");
	745	my $split_script_filepath = &util::filename_cat($gli_help_directory, "splithelpdocument.pl");
	746
	747	my $target_file_directory = &util::filename_cat($gli_help_directory, $target_language_code);
	748	$target_file_directory = $target_file_directory."/";
[24627]	749
[18460]	750	my $target_filepath = &util::filename_cat($gsdl_root_directory, $target_file);
[28991]	751
	752	# if gli/help/nl doesn't exist, create it by copying over gli/help/en/help.xml, then process the copied file
	753	my ($tailname, $glihelp_lang_dir, $suffix) = &File::Basename::fileparse($target_filepath, "\\.[^\\.]+\$");
	754	if(!&FileUtils::directoryExists($glihelp_lang_dir)) {
	755
	756	# copy across the gli/help/en/help.xml into a new folder for the new language gli/help/<newlang>
	757	my $en_glihelp_dir = &util::filename_cat($gli_help_directory, "en");
[28992]	758	my $en_helpxml_file = &util::filename_cat($en_glihelp_dir, "$tailname$suffix"); #$tailname$suffix="help.xml"
[28991]	759	&FileUtils::copyFilesRecursiveNoSVN($en_helpxml_file, $glihelp_lang_dir);
	760
	761	# In gli/help/<newlang>/help.xml, replace all occurrences of
	762	# <Text id="1">This text in en will be removed for new langcode</Text>
	763	# with <!-- Missing translation: 1 -->
	764	open(FIN,"<$target_filepath") or &throw_fatal_error("Could not open $target_filepath for READING after creating it");
	765	my $help_xml_contents;
	766	# Read in the entire contents of the file in one hit
	767	sysread(FIN, $help_xml_contents, -s FIN);
	768	close(FIN);
[24627]	769
[28991]	770	$help_xml_contents =~ s@<Text id="([^"]+?)">(.*?)</Text>@<!-- Missing translation: $1 -->@sg;
	771
	772	open(FOUT, ">$target_filepath") or &throw_fatal_error("Could not open $target_filepath for WRITING after creating it");
	773	print FOUT $help_xml_contents;
	774	close(FOUT);
	775	}
	776
[24627]	777	my $perl_exec = &util::get_perl_exec();
	778	my $java_exec = "java";
	779	if(defined($ENV{'JAVA_HOME'}) && $ENV{'JAVA_HOME'} ne ""){
	780	$java_exec = &util::filename_cat($ENV{'JAVA_HOME'}, "bin", "java");
	781	}
[28991]	782
	783	#my $cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	784	my $cmd = "$java_exec -DGSDLHOME=$gsdl_root_directory -cp $classpath ApplyXSLT $target_language_code $gen_many_html_xsl_filepath $target_filepath \| \"$perl_exec\" -S $split_script_filepath $target_file_directory";
	785	#&throw_fatal_error("RAN gti command: $cmd");
[18460]	786	my $response = `$cmd`;
[28991]	787
	788	#$cmd = "$java_exec -cp $classpath:$classpath/xalan.jar ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
	789	$cmd = "$java_exec -cp $classpath -DGSDLHOME=$gsdl_root_directory ApplyXSLT $target_language_code $gen_index_xml_xsl_filepath $target_filepath > " . $target_file_directory . "help_index.xml"; # 2>/dev/null";
[18460]	790	$response = `$cmd`;
[28991]	791
	792	# create a gti/tmp folder, if one doesn't already exist, and store the downloadable zip file in there
	793	my $tmpdir = &util::filename_cat($gsdl_root_directory, "tmp");
	794	if(!&FileUtils::directoryExists($tmpdir)) {
	795	&FileUtils::makeDirectory($tmpdir);
	796	}
[28977]	797	#my $zip_file_path = "/greenstone/custom/gti/" . $target_language_code . "_GLIHelp.zip";
[28991]	798	my $zip_file_path = &util::filename_cat($tmpdir, $target_language_code . "_GLIHelp.zip");
[18460]	799	$cmd = "zip -rj $zip_file_path $target_file_directory -i \.htm \.xml";
[28991]	800
[18460]	801	$response = `$cmd`;
[10019]	802	}
	803
	804
	805	sub get_translation_configuration
	806	{
	807	# Get the code of the target language
	808	my $target_language_code = shift(@_);
	809	# Get the key of the file to translate
	810	my $translation_file_key = shift(@_);
[24627]	811
[10019]	812	# Read the translation data from the gti.cfg file
	813	my ($source_file, $target_file, $translation_file_type) =
	814	&get_translation_data_for($target_language_code, $translation_file_key);
[24627]	815
[10019]	816	# Check that the file to translate is defined in the gti.cfg file
	817	if (!$source_file \|\| !$target_file \|\| !$translation_file_type) {
[24627]	818	&throw_fatal_error("Missing or incomplete specification for translation file \"$translation_file_key\" in gti.pl.");
[10019]	819	}
[24627]	820
[10019]	821	# Check that the source file exists
	822	my $source_file_path = &util::filename_cat($gsdl_root_directory, $source_file);
	823	if (!-e $source_file_path) {
[24627]	824	&throw_fatal_error("Source file $source_file_path does not exist.");
[10019]	825	}
[24627]	826
[10019]	827	# Check that the source file is up to date
[11104]	828	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[10019]	829	# unless ($translation_file_is_not_in_cvs) {
[14258]	830	#my $source_file_cvs_status = `cd $gsdl_root_directory; cvs -d $anonymous_cvs_root update $source_file 2>/dev/null`;
[24627]	831	my $source_file_cvs_status = `cd $gsdl_root_directory; svn status $source_file 2>/dev/null`;
[10019]	832	if ($source_file_cvs_status =~ /^C /) {
	833	&throw_fatal_error("Source file $source_file_path conflicts with the repository.");
	834	}
	835	if ($source_file_cvs_status =~ /^M /) {
	836	&throw_fatal_error("Source file $source_file_path contains uncommitted changes.");
	837	}
	838	# }
[24627]	839
[10019]	840	return ($source_file, $target_file, $translation_file_type);
	841	}
	842
	843
	844	sub get_translation_data_for
	845	{
	846	my ($target_language_code, $translation_file_key) = @_;
[24627]	847
[10019]	848	foreach my $translation_file (@$gti_translation_files) {
[24627]	849	# If this isn't the correct translation file, move onto the next one
	850	next if ($translation_file_key ne $translation_file->{'key'});
	851
	852	# Resolve the target language file
	853	my $target_language_file = $translation_file->{'target_file'};
	854	if ($target_language_file =~ /(\{.+\;.+\})/) {
	855	my $unresolved_target_language_file_part = $1;
	856
	857	# Check for a special case for the target language code
	858	if ($unresolved_target_language_file_part =~ /(\{\|\;)$target_language_code:([^\;]+)(\;\|\})/) {
	859	my $resolved_target_language_file_part = $2;
	860	$target_language_file =~ s/$unresolved_target_language_file_part/$resolved_target_language_file_part/;
	861	}
	862	# Otherwise use the last part as the default value
	863	else {
	864	my ($default_target_language_file_part) = $unresolved_target_language_file_part =~ /([^\;]+)\}/;
	865	$target_language_file =~ s/$unresolved_target_language_file_part/\{$default_target_language_file_part\}/;
[10019]	866	}
[10041]	867	}
[24627]	868
[10041]	869	# Resolve instances of {iso_639_1_target_language_name}
	870	my $iso_639_1_target_language_name = $iso639::fromiso639{$target_language_code};
	871	$iso_639_1_target_language_name =~ tr/A-Z/a-z/ if $iso_639_1_target_language_name;
	872	$target_language_file =~ s/\{iso_639_1_target_language_name\}/$iso_639_1_target_language_name/g;
[24627]	873
[10041]	874	# Resolve instances of {target_language_code}
	875	$target_language_file =~ s/\{target_language_code\}/$target_language_code/g;
[24627]	876
[10041]	877	return ($translation_file->{'source_file'}, $target_language_file, $translation_file->{'file_type'});
[24627]	878	}
[10019]	879
[24627]	880	return ();
[10019]	881	}
	882
	883
	884	sub read_file_lines
	885	{
	886	my ($file_path) = @_;
[24627]	887
[10019]	888	if (!open(FILE_IN, "<$file_path")) {
[24627]	889	&log_message("Note: Could not open file $file_path.");
	890	return ();
[10019]	891	}
	892	my @file_lines = <FILE_IN>;
	893	close(FILE_IN);
[24627]	894
[10019]	895	return @file_lines;
	896	}
	897
	898
	899	sub build_key_to_line_mapping
	900	{
	901	my ($file_lines, $translation_file_type) = @_;
	902	eval "return &build_key_to_line_mapping_for_${translation_file_type}(\@\$file_lines)";
	903	}
	904
	905
	906	sub build_key_to_text_mapping
	907	{
	908	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	909
[10019]	910	my %key_to_text_mapping = ();
	911	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	912	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	913	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	914
	915	my $chunk_text = @$file_lines[$chunk_starting_line];
	916	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	917	$chunk_text .= @$file_lines[$l];
	918	}
	919
	920	# Map from chunk key to text
	921	eval "\$key_to_text_mapping{\${chunk_key}} = &import_chunk_from_${translation_file_type}(\$chunk_text)";
[10019]	922	}
[24627]	923
[10019]	924	return %key_to_text_mapping;
	925	}
	926
	927
	928	sub build_key_to_last_update_date_mapping
	929	{
	930	my ($file, $file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	931
[10019]	932	# If the files aren't in CVS then we can't tell anything about what needs updating
	933	# return () if ($translation_file_is_not_in_cvs);
[24627]	934
[10019]	935	# Build a mapping from key to CVS date
	936	# Need to be careful with this mapping because the chunk keys won't necessarily all be valid
	937	my %key_to_cvs_date_mapping = &build_key_to_cvs_date_mapping($file, $translation_file_type);
[24627]	938
[10019]	939	# Build a mapping from key to comment date
[12483]	940	my %key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping($file_lines, $key_to_line_mapping, $translation_file_type);
[24627]	941
[10019]	942	# Build a mapping from key to last update date (the latter of the CVS date and comment date)
	943	my %key_to_last_update_date_mapping = ();
	944	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	945	# Use the CVS date as a starting point
	946	my $chunk_cvs_date = $key_to_cvs_date_mapping{$chunk_key};
	947	$key_to_last_update_date_mapping{$chunk_key} = $chunk_cvs_date;
	948
	949	# If a comment date exists and it is after the CVS date, use that instead
[14258]	950	# need to convert the comment date format to SVN format
[24627]	951	my $chunk_gti_comment = $key_to_gti_comment_mapping{$chunk_key};
	952	if (defined($chunk_gti_comment) && $chunk_gti_comment =~ /(\d?\d-\D\D\D-\d\d\d\d)/) {
	953	my $chunk_comment_date = $1;
	954	if ((!defined($chunk_cvs_date) \|\| &is_date_after($chunk_comment_date, $chunk_cvs_date))) {
	955	$key_to_last_update_date_mapping{$chunk_key} = $chunk_comment_date;
	956	}
	957	}
[10019]	958	}
[24627]	959
[10019]	960	return %key_to_last_update_date_mapping;
	961	}
	962
	963
	964	sub build_key_to_cvs_date_mapping
	965	{
	966	my ($filename, $translation_file_type) = @_;
[24627]	967
	968	# Use SVN to annotate each line of the file with the date it was last edited
[10019]	969	# The "2>/dev/null" is very important! If it is missing this will never return when run from the receptionist
[24627]	970	my $cvs_annotated_file = `cd $gsdl_root_directory; svn annotate -v --force $filename 2>/dev/null`;
[14258]	971
[10019]	972	my @cvs_annotated_file_lines = split(/\n/, $cvs_annotated_file);
[24627]	973
[10019]	974	my @cvs_annotated_file_lines_date = ();
	975	foreach my $cvs_annotated_file_line (@cvs_annotated_file_lines) {
[24627]	976	# Extract the date from the SVN annotation at the front
	977	# svn format : 2007-07-16
[14258]	978	$cvs_annotated_file_line =~ s/^\s+\S+\s+\S+\s(\S+)//;
	979
	980	push(@cvs_annotated_file_lines_date, $1);
	981
	982	# trim extra date information in svn annotation format
	983	# 15:42:49 +1200 (Wed, 21 Jun 2006)
	984	$cvs_annotated_file_line =~ s/^\s+\S+\s\S+\s\((.+?)\)\s//;
	985	}
	986
[10019]	987	# Build a key to line mapping for the CVS annotated file, for matching the chunk key to the CVS date
	988	my %key_to_line_mapping = &build_key_to_line_mapping(\@cvs_annotated_file_lines, $translation_file_type);
[24627]	989
[10019]	990	my %key_to_cvs_date_mapping = ();
	991	foreach my $chunk_key (keys(%key_to_line_mapping)) {
[24627]	992	my $chunk_starting_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[0];
	993	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping{$chunk_key}))[1];
	994
	995	# Find the date this chunk was last edited, from the CVS annotation
	996	my $chunk_date = $cvs_annotated_file_lines_date[$chunk_starting_line];
	997	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	998	if (&is_date_after($cvs_annotated_file_lines_date[$l], $chunk_date)) {
	999	# This part of the chunk has been updated more recently
	1000	$chunk_date = $cvs_annotated_file_lines_date[$l];
	1001
	1002	}
	1003	}
	1004
	1005	# Map from chunk key to CVS date
	1006	$key_to_cvs_date_mapping{$chunk_key} = $chunk_date;
[10019]	1007	}
[24627]	1008
[10019]	1009	return %key_to_cvs_date_mapping;
	1010	}
	1011
	1012
[12483]	1013	sub build_key_to_gti_comment_mapping
[10019]	1014	{
	1015	my ($file_lines, $key_to_line_mapping, $translation_file_type) = @_;
[24627]	1016
[12483]	1017	my %key_to_gti_comment_mapping = ();
[10019]	1018	foreach my $chunk_key (keys(%$key_to_line_mapping)) {
[24627]	1019	my $chunk_starting_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[0];
	1020	my $chunk_finishing_line = (split(/-/, $key_to_line_mapping->{$chunk_key}))[1];
	1021
	1022	my $chunk_text = @$file_lines[$chunk_starting_line];
	1023	for (my $l = ($chunk_starting_line + 1); $l <= $chunk_finishing_line; $l++) {
	1024	$chunk_text .= @$file_lines[$l];
	1025	}
	1026
	1027	# Map from chunk key to GTI comment
	1028	my $chunk_gti_comment;
	1029	eval "\$chunk_gti_comment = &get_${translation_file_type}_chunk_gti_comment(\$chunk_text)";
	1030	$key_to_gti_comment_mapping{$chunk_key} = $chunk_gti_comment if (defined($chunk_gti_comment));
[10019]	1031	}
[24627]	1032
[12483]	1033	return %key_to_gti_comment_mapping;
[10019]	1034	}
	1035
	1036
	1037	sub determine_chunks_requiring_translation
	1038	{
	1039	my $source_file_key_to_text_mapping = shift(@_);
	1040	my $target_file_key_to_text_mapping = shift(@_);
[24627]	1041
[10019]	1042	# Chunks needing translation are those in the source file with no translation in the target file
	1043	my @target_file_keys_requiring_translation = ();
	1044	foreach my $chunk_key (keys(%$source_file_key_to_text_mapping)) {
[24627]	1045	if ($source_file_key_to_text_mapping->{$chunk_key} && !$target_file_key_to_text_mapping->{$chunk_key}) {
	1046	# &log_message("Chunk with key $chunk_key needs translating.");
	1047	push(@target_file_keys_requiring_translation, $chunk_key);
	1048	}
[10019]	1049	}
[24627]	1050
[10019]	1051	return @target_file_keys_requiring_translation;
	1052	}
	1053
	1054
	1055	sub determine_chunks_requiring_updating
	1056	{
	1057	my $source_file_key_to_last_update_date_mapping = shift(@_);
	1058	my $target_file_key_to_last_update_date_mapping = shift(@_);
[24627]	1059
[10019]	1060	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	1061	my @target_file_keys_requiring_updating = ();
	1062	foreach my $chunk_key (keys(%$source_file_key_to_last_update_date_mapping)) {
[24627]	1063	my $source_chunk_last_update_date = $source_file_key_to_last_update_date_mapping->{$chunk_key};
	1064	my $target_chunk_last_update_date = $target_file_key_to_last_update_date_mapping->{$chunk_key};
[14258]	1065
	1066	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
[24627]	1067
[14258]	1068	if (defined($target_chunk_last_update_date) && &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
[24627]	1069	# &log_message("Chunk with key $chunk_key needs updating.");
	1070	push(@target_file_keys_requiring_updating, $chunk_key);
	1071	}
[10019]	1072	}
[24627]	1073
[10019]	1074	return @target_file_keys_requiring_updating;
	1075	}
	1076
	1077
	1078	sub is_chunk_automatically_translated
	1079	{
	1080	my ($chunk_key, $translation_file_type) = @_;
	1081	eval "return &is_${translation_file_type}_chunk_automatically_translated(\$chunk_key)";
	1082	}
	1083
	1084
	1085	sub make_text_xml_safe
	1086	{
	1087	my $text = shift(@_);
[11452]	1088	$text =~ s/\&/\&amp\;/g;
	1089	$text =~ s/\&amp\;lt\;/\&amp\;amp\;lt\;/g;
	1090	$text =~ s/\&amp\;gt\;/\&amp\;amp\;gt\;/g;
[11498]	1091	$text =~ s/\&amp\;rarr\;/\&amp\;amp\;rarr\;/g;
	1092	$text =~ s/\&amp\;mdash\;/\&amp\;amp\;mdash\;/g;
[10019]	1093	$text =~ s/</\&lt\;/g;
	1094	$text =~ s/>/\&gt\;/g;
	1095	return $text;
	1096	}
	1097
	1098
[11448]	1099	sub unmake_text_xml_safe
	1100	{
	1101	my $text = shift(@_);
	1102	$text =~ s/\&lt\;/</g;
	1103	$text =~ s/\&gt\;/>/g;
	1104	$text =~ s/\&amp\;/\&/g;
	1105	return $text;
	1106	}
	1107
	1108
[10019]	1109	# Returns 1 if $date1 is after $date2, 0 otherwise
[14258]	1110	sub is_date_after_cvs
[10019]	1111	{
	1112	my ($date1, $date2) = @_;
	1113	my %months = ("Jan", 1, "Feb", 2, "Mar", 3, "Apr", 4, "May", 5, "Jun", 6,
[24627]	1114	"Jul", 7, "Aug", 8, "Sep", 9, "Oct", 10, "Nov", 11, "Dec", 12);
	1115
[14258]	1116	if(!defined $date1) {
	1117	return 1;
	1118	}
[24627]	1119
[10019]	1120	my @date1parts = split(/-/, $date1);
	1121	my @date2parts = split(/-/, $date2);
[24627]	1122
[10019]	1123	# Compare year - nasty because we have rolled over into a new century
	1124	my $year1 = $date1parts[2];
	1125	if ($year1 < 80) {
[14258]	1126	$year1 += 2000;
[10019]	1127	}
	1128	my $year2 = $date2parts[2];
	1129	if ($year2 < 80) {
[14258]	1130	$year2 += 2000;
[10019]	1131	}
[24627]	1132
[10019]	1133	# Compare year
	1134	if ($year1 > $year2) {
[24627]	1135	return 1;
[10019]	1136	}
	1137	elsif ($year1 == $year2) {
[24627]	1138	# Year is the same, so compare month
	1139	if ($months{$date1parts[1]} > $months{$date2parts[1]}) {
	1140	return 1;
	1141	}
	1142	elsif ($months{$date1parts[1]} == $months{$date2parts[1]}) {
	1143	# Month is the same, so compare day
	1144	if ($date1parts[0] > $date2parts[0]) {
	1145	return 1;
	1146	}
	1147	}
[10019]	1148	}
[24627]	1149
[10019]	1150	return 0;
	1151	}
	1152
[14258]	1153	sub is_date_after
	1154	{
	1155	my ($date1, $date2) = @_;
	1156
	1157	if(!defined $date1) {
[24627]	1158	return 1;
[14258]	1159	}
	1160	if(!defined $date2) {
[24627]	1161	return 0;
[14258]	1162	}
	1163
	1164	# 16-Aug-2006
	1165	if($date1=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1166	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1167	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1168	$date1=$3 . "-" . $months{$2} . "-" . $1;
	1169	# print "** converted date1: $date1\n";
[14258]	1170	}
	1171	if($date2=~ /(\d+?)-(\S\S\S)-(\d\d\d\d)/){
[24627]	1172	my %months = ("Jan", "01", "Feb", "02", "Mar", "03", "Apr", "04", "May", "05", "Jun", "06",
	1173	"Jul", "07", "Aug", "08", "Sep", "09", "Oct", "10", "Nov", "11", "Dec", "12");
	1174	$date2=$3 . "-" . $months{$2} . "-" . $1;
	1175	# print "** converted date2: $date2\n";
[14258]	1176	}
	1177
	1178
	1179	# 2006-08-16
	1180	my @date1parts = split(/-/, $date1);
	1181	my @date2parts = split(/-/, $date2);
	1182
	1183	# Compare year
	1184	if ($date1parts[0] > $date2parts[0]) {
[24627]	1185	return 1;
[14258]	1186	}
	1187	elsif ($date1parts[0] == $date2parts[0]) {
[24627]	1188	# Year is the same, so compare month
	1189	if ($date1parts[1] > $date2parts[1]) {
	1190	return 1;
	1191	}
	1192	elsif ($date1parts[1] == $date2parts[1]) {
	1193	# Month is the same, so compare day
	1194	if ($date1parts[2] > $date2parts[2]) {
	1195	return 1;
	1196	}
	1197	}
[14258]	1198	}
	1199
	1200	return 0;
	1201	}
[10019]	1202
[14258]	1203
[18460]	1204	sub create_xml_response_for_chunks_requiring_work
	1205	{
	1206	my ($translation_file_key, $target_file, $total_num_chunks, $target_files_keys_requiring_translation, $target_files_keys_requiring_updating, $num_chunks_to_return, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
[24627]	1207
[18460]	1208	# Form an XML response to the command
	1209	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1210	$xml_response .= "<GTIResponse>\n";
	1211	$xml_response .= " <TranslationFile"
	1212	. " key=\"" . $translation_file_key . "\""
	1213	. " target_file_path=\"" . $target_file . "\""
	1214	. " num_chunks_translated=\"" . ($total_num_chunks - scalar(@$target_files_keys_requiring_translation)) . "\""
	1215	. " num_chunks_requiring_translation=\"" . scalar(@$target_files_keys_requiring_translation) . "\""
	1216	. " num_chunks_requiring_updating=\"" . scalar(@$target_files_keys_requiring_updating) . "\"\/>\n";
[24627]	1217
[18460]	1218	# Do chunks requiring translation first
	1219	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_translation)) {
[24627]	1220	$xml_response .= " <ChunksRequiringTranslation size=\"" . scalar(@$target_files_keys_requiring_translation) . "\">\n";
[18460]	1221	}
	1222	else {
[24627]	1223	$xml_response .= " <ChunksRequiringTranslation size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1224	}
[24627]	1225
[18460]	1226	my @sorted_chunk_keys = sort (@$target_files_keys_requiring_translation);
	1227	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1228	last if ($num_chunks_to_return == 0);
	1229
	1230	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1231	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1232
	1233	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1234	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1235	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1236	$xml_response .= " </Chunk>\n";
	1237
	1238	$num_chunks_to_return--;
	1239	}
[18460]	1240
	1241	$xml_response .= " </ChunksRequiringTranslation>\n";
[24627]	1242
[18460]	1243	# Then do chunks requiring updating
	1244	if ($num_chunks_to_return > scalar(@$target_files_keys_requiring_updating)) {
[24627]	1245	$xml_response .= " <ChunksRequiringUpdating size=\"" . scalar(@$target_files_keys_requiring_updating) . "\">\n";
[18460]	1246	}
	1247	else {
[24627]	1248	$xml_response .= " <ChunksRequiringUpdating size=\"" . $num_chunks_to_return . "\">\n";
[18460]	1249	}
[24627]	1250
[18460]	1251	# foreach my $chunk_key (@target_file_keys_requiring_updating) {
	1252	@sorted_chunk_keys = sort (@$target_files_keys_requiring_updating);
	1253	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1254	last if ($num_chunks_to_return == 0);
	1255
	1256	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1257	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1258	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1259	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1260
	1261	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1262	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1263	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1264	$xml_response .= " </Chunk>\n";
	1265
	1266	$num_chunks_to_return--;
[18460]	1267	}
[24627]	1268
[18460]	1269	$xml_response .= " </ChunksRequiringUpdating>\n";
[24627]	1270
[18460]	1271	$xml_response .= "</GTIResponse>\n";
[24627]	1272
[18460]	1273	return $xml_response;
	1274	}
	1275
[25249]	1276	sub create_xml_response_for_uptodate_chunks
	1277	{
	1278	my ($translation_file_key, $target_file, $uptodate_target_files_keys, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping, $source_files_key_to_last_update_date_mapping, $target_files_key_to_last_update_date_mapping) = @_;
	1279
	1280	# Form an XML response to the command
	1281	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1282	$xml_response .= "<GTIResponse>\n";
	1283	$xml_response .= " <TranslationFile"
	1284	. " key=\"" . $translation_file_key . "\""
	1285	. " target_file_path=\"" . $target_file . "\""
	1286	. " num_chunks_uptodate=\"" . scalar(@$uptodate_target_files_keys) . "\"\/>\n";
	1287
	1288
	1289	# Then do chunks requiring updating
	1290	$xml_response .= " <UptodateChunks size=\"" . scalar(@$uptodate_target_files_keys) . "\">\n";
	1291
	1292
	1293	# foreach my $chunk_key (@uptodate_target_file_keys) {
	1294	my @sorted_chunk_keys = sort (@$uptodate_target_files_keys);
	1295	foreach my $chunk_key (@sorted_chunk_keys) {
	1296
	1297	my $source_file_chunk_date = $source_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1298	my $source_file_chunk_text = &make_text_xml_safe($source_files_key_to_text_mapping->{$chunk_key});
	1299	my $target_file_chunk_date = $target_files_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1300	my $target_file_chunk_text = &make_text_xml_safe($target_files_key_to_text_mapping->{$chunk_key});
	1301
	1302	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1303	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1304	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1305	$xml_response .= " </Chunk>\n";
[18460]	1306
[25249]	1307	}
	1308
	1309	$xml_response .= " </UptodateChunks>\n";
	1310
	1311	$xml_response .= "</GTIResponse>\n";
	1312
	1313	return $xml_response;
	1314	}
	1315
[18460]	1316	sub create_xml_response_for_all_chunks
	1317	{
	1318	my ($translation_file_key, $target_file, $source_file_key_to_text_mapping, $target_file_key_to_text_mapping, $source_file_key_to_last_update_date_mapping, $target_file_key_to_last_update_date_mapping) = @_;
[24627]	1319
[18460]	1320	# Form an XML response to the command
	1321	my $xml_response = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
	1322	$xml_response .= "<GTIResponse>\n";
	1323	$xml_response .= " <TranslationFile"
	1324	. " key=\"" . $translation_file_key . "\""
	1325	. " target_file_path=\"" . $target_file . "\"\/>\n";
	1326
	1327	# Do all the chunks
	1328	$xml_response .= " <Chunks size=\"" . scalar(keys(%$source_file_key_to_text_mapping)) . "\">\n";
[24627]	1329
[18460]	1330	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
	1331	foreach my $chunk_key (@sorted_chunk_keys) {
[24627]	1332	my $source_file_chunk_date = $source_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1333	my $source_file_chunk_text = &make_text_xml_safe($source_file_key_to_text_mapping->{$chunk_key});
	1334
	1335	$xml_response .= " <Chunk key=\"" . &make_text_xml_safe($chunk_key) . "\">\n";
	1336	$xml_response .= " <SourceFileText date=\"$source_file_chunk_date\">$source_file_chunk_text</SourceFileText>\n";
	1337	if (defined($target_file_key_to_text_mapping->{$chunk_key})) {
	1338	my $target_file_chunk_date = $target_file_key_to_last_update_date_mapping->{$chunk_key} \|\| "";
	1339	my $target_file_chunk_text = &make_text_xml_safe($target_file_key_to_text_mapping->{$chunk_key});
	1340	$xml_response .= " <TargetFileText date=\"$target_file_chunk_date\">$target_file_chunk_text</TargetFileText>\n";
	1341	}
	1342	else {
	1343	$xml_response .= " <TargetFileText></TargetFileText>\n";
	1344	}
	1345
	1346	$xml_response .= " </Chunk>\n";
[18460]	1347	}
	1348	$xml_response .= " </Chunks>\n";
	1349
	1350	$xml_response .= "</GTIResponse>\n";
	1351	return $xml_response;
	1352	}
	1353
	1354
	1355
[10019]	1356	# ==========================================================================================
	1357	# MACROFILE FUNCTIONS
	1358
	1359	sub build_key_to_line_mapping_for_macrofile
	1360	{
	1361	my (@file_lines) = @_;
[24627]	1362
[10019]	1363	my $macro_package;
	1364	my %chunk_key_to_line_mapping = ();
	1365	# Process the contents of the file, line by line
	1366	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1367	my $line = $file_lines[$i];
	1368	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1369
	1370	# Check if a new package is being defined
	1371	if ($line =~ m/^package\s+(.+)/) {
	1372	$macro_package = $1;
[10041]	1373	}
[24627]	1374
	1375	# Line contains a macro name
	1376	elsif ($line =~ m/^(_\w+_)/) {
	1377	my $macro_key = $1;
	1378	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1379
	1380	# While there is still text of the macro to go...
	1381	my $startline = $i;
	1382	while ($line !~ /\}$/) {
	1383	$i++;
	1384	if ($i == scalar(@file_lines)) {
	1385	&throw_fatal_error("Could not find end of macro $macro_key.");
	1386	}
	1387	$line = $file_lines[$i];
	1388	$line =~ s/\s*([^\\]\#[^\}]+)?$//; # Remove any comments and nasty whitespace
	1389	}
	1390
[10019]	1391	# The chunk key consists of the package name and the macro key
	1392	my $chunk_key = $macro_package . "." . $macro_key;
	1393	# Map from chunk key to line
	1394	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1395	}
[24627]	1396
[10019]	1397	# Icon: line in format ## "image text" ## image_type ## macro_name ##
	1398	elsif ($line =~ m/^\#\# .* \#\# .* \#\# (.*) \#\#/) {
[24627]	1399	# The chunk key consists of package name and macro key
	1400	my $chunk_key = $macro_package . "." . $1;
	1401	# Map from chunk key to line
	1402	$chunk_key_to_line_mapping{$chunk_key} = $i . "-" . $i;
	1403	}
	1404	}
[10019]	1405
[24627]	1406	return %chunk_key_to_line_mapping;
[10019]	1407	}
	1408
	1409
	1410	sub import_chunk_from_macrofile
	1411	{
	1412	my ($chunk_text) = @_;
[24627]	1413
[10019]	1414	# Is this an icon macro??
	1415	if ($chunk_text =~ /^\#\# (.*)/) {
[24627]	1416	# Extract image macro text
	1417	$chunk_text =~ /^\#\#\s+([^\#]+)\s+\#\#/;
	1418	$chunk_text = $1;
	1419
[10019]	1420	# Remove enclosing quotes
	1421	$chunk_text =~ s/^\"//;
	1422	$chunk_text =~ s/\"$//;
[24627]	1423	}
[10019]	1424
[24627]	1425	# No, so it must be a text macro
	1426	else {
[10019]	1427	# Remove macro key
	1428	$chunk_text =~ s/^_([^_]+)_(\s*)//;
[24627]	1429
[10019]	1430	# Remove language specifier
	1431	$chunk_text =~ s/^\[l=.\](\s)//;
[24627]	1432
[10019]	1433	# Remove braces enclosing text
	1434	$chunk_text =~ s/^{(\s)((.\|\n))}(\s)(\#.+\s)?/$2/;
[24627]	1435	}
[10019]	1436
[24627]	1437	return $chunk_text;
[10019]	1438	}
	1439
	1440
[12483]	1441	sub get_macrofile_chunk_gti_comment
[10019]	1442	{
	1443	my ($chunk_text) = @_;
[24627]	1444
[10019]	1445	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1446	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1447	return $1;
	1448	}
[10019]	1449
[24627]	1450	return undef;
[10019]	1451	}
	1452
	1453
	1454	sub is_macrofile_chunk_automatically_translated
	1455	{
	1456	my ($chunk_key) = @_;
[24627]	1457
[10019]	1458	# The _httpiconX_, _widthX_ and _heightX_ image macros are automatically translated
	1459	if ($chunk_key =~ /\._(httpicon\|width\|height)/) {
[24627]	1460	return 1;
[10019]	1461	}
[24627]	1462
[10019]	1463	return 0;
	1464	}
	1465
	1466
	1467	# Use the source file to generate a target file that is formatted the same
	1468	sub write_translated_macrofile
	1469	{
	1470	my $source_file = shift(@_); # Not used
[11321]	1471	my @source_file_lines = @{shift(@_)};
[10019]	1472	my $source_file_key_to_text_mapping = shift(@_);
	1473	my $target_file = shift(@_);
[11321]	1474	my @target_file_lines = @{shift(@_)};
[10019]	1475	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1476	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1477	my $target_language_code = shift(@_);
[24627]	1478
[10126]	1479	# Build a mapping from source file line to chunk key
[11321]	1480	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@source_file_lines);
[10019]	1481	my %source_file_line_to_key_mapping = ();
	1482	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1483	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1484	}
[10126]	1485	my @source_file_line_keys = (sort sort_by_line (keys(%source_file_line_to_key_mapping)));
	1486	my $source_file_line_number = 0;
[24627]	1487
[10126]	1488	# Build a mapping from target file line to chunk key
[11321]	1489	my %target_file_key_to_line_mapping = &build_key_to_line_mapping_for_macrofile(@target_file_lines);
[10126]	1490	my %target_file_line_to_key_mapping = ();
	1491	foreach my $chunk_key (keys(%target_file_key_to_line_mapping)) {
[24627]	1492	$target_file_line_to_key_mapping{$target_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10126]	1493	}
	1494	my @target_file_line_keys = (sort sort_by_line (keys(%target_file_line_to_key_mapping)));
[24627]	1495
[10019]	1496	# Write the new target file
	1497	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1498	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1499	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1500	}
[24627]	1501
[10126]	1502	# Use the header from the target file, to keep language and author information
	1503	if (scalar(@target_file_line_keys) > 0) {
[24627]	1504	my $target_file_line_number = 0;
	1505	my $target_file_chunk_starting_line_number = (split(/-/, $target_file_line_keys[0]))[0];
	1506	while ($target_file_line_number < $target_file_chunk_starting_line_number) {
	1507	my $target_file_line = $target_file_lines[$target_file_line_number];
	1508	last if ($target_file_line =~ /^\# -- Missing translation: /); # We don't want to get into the macros
	1509	print TARGET_FILE $target_file_line;
	1510	$target_file_line_number++;
	1511	}
	1512
	1513	$source_file_line_number = (split(/-/, $source_file_line_keys[0]))[0];
[10126]	1514	}
[24627]	1515
[10019]	1516	# Model the new target file on the source file, with the target file translations
[10126]	1517	foreach my $line_key (@source_file_line_keys) {
[24627]	1518	# Fill in the gaps before this chunk starts
	1519	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1520	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1521	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1522	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1523	$source_file_line_number++;
	1524	}
	1525	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1526
	1527	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1528	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1529	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1530
	1531	my $macrofile_key = $chunk_key;
	1532	$macrofile_key =~ s/^(.+?)\.//;
	1533
	1534	# If no translation exists for this chunk, show this, and move on
	1535	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1536	print TARGET_FILE "# -- Missing translation: $macrofile_key\n";
	1537	next;
	1538	}
	1539
	1540	# Grab the source chunk text
	1541	my $source_file_chunk = $source_file_lines[$source_file_chunk_starting_line_number];
	1542	for (my $l = ($source_file_chunk_starting_line_number + 1); $l <= $source_file_chunk_finishing_line_number; $l++) {
	1543	$source_file_chunk .= $source_file_lines[$l];
	1544	}
	1545
	1546	# Is this an icon macro??
	1547	if ($source_file_chunk =~ /^\#\# (.*)/) {
	1548	# Escape any newline and question mark characters so the source text is replaced correctly
	1549	$source_file_chunk_text =~ s/\\/\\\\/g;
[10019]	1550	$source_file_chunk_text =~ s/\?/\\\?/g;
[24627]	1551
[10019]	1552	# Build the new target chunk from the source chunk
	1553	my $target_file_chunk = $source_file_chunk;
	1554	$target_file_chunk =~ s/$source_file_chunk_text/$target_file_chunk_text/;
[10050]	1555	$target_file_chunk =~ s/(\s)*$//;
[10019]	1556	print TARGET_FILE "$target_file_chunk";
	1557	}
[24627]	1558
[10019]	1559	# No, it is just a normal text macro
	1560	else {
	1561	print TARGET_FILE "$macrofile_key [l=$target_language_code] {$target_file_chunk_text}";
	1562	}
[24627]	1563
[12483]	1564	# Add the "updated" comment, if one exists
	1565	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1566	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
[10050]	1567	}
	1568	print TARGET_FILE "\n";
[24627]	1569	}
[10019]	1570
[24627]	1571	close(TARGET_FILE);
[10019]	1572	}
	1573
	1574
	1575	sub sort_by_line
	1576	{
	1577	return ((split(/-/, $a))[0] <=> (split(/-/, $b))[0]);
	1578	}
	1579
	1580
	1581	# ==========================================================================================
	1582	# RESOURCE BUNDLE FUNCTIONS
	1583
[29456]	1584	# need to handle multi-line properties. A multiline ends on \ if it continues over the next line
[10019]	1585	sub build_key_to_line_mapping_for_resource_bundle
	1586	{
	1587	my (@file_lines) = @_;
[24627]	1588
[11321]	1589	my %chunk_key_to_line_mapping = ();
[29456]	1590
	1591	my $chunk_key;
	1592	my $startindex = -1;
	1593
[10019]	1594	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1595	my $line = $file_lines[$i];
	1596	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1597
	1598	# Line contains a dictionary string
	1599	if ($line =~ /^(\S+?)[:\|=](.*)$/) {
[29456]	1600	$chunk_key = $1;
	1601	$startindex = $i;
	1602	}
	1603	if ($startindex != -1) {
	1604	if($line !~ m/\\$/) { # line finished
	1605	# $i keeps track of the line at which this property (chunk_key) finishes
	1606
[24627]	1607	# Map from chunk key to line
[29456]	1608	$chunk_key_to_line_mapping{$chunk_key} = $startindex . "-" . $i;
	1609	$startindex = -1;
	1610	$chunk_key = "";
	1611	}
	1612	}
[10019]	1613	}
[24627]	1614
[11321]	1615	return %chunk_key_to_line_mapping;
[10019]	1616	}
	1617
	1618
	1619	sub import_chunk_from_resource_bundle
	1620	{
	1621	my ($chunk_text) = @_;
[24627]	1622
[10019]	1623	# Simple: just remove string key
[29456]	1624	$chunk_text =~ s/^(\S+?)[:\|=](\s*)//s;
	1625	$chunk_text =~ s/(\s*)$//s; # Remove any nasty whitespace, carriage returns etc.
	1626	$chunk_text =~ s/(\s)\#\s+Updated\s+(\d?\d-\D\D\D-\d\d\d\d.)\s*$//is;
[24627]	1627
[10019]	1628	return $chunk_text;
	1629	}
	1630
	1631
[12483]	1632	sub get_resource_bundle_chunk_gti_comment
[10019]	1633	{
	1634	my ($chunk_text) = @_;
[24627]	1635
[10019]	1636	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1637	if ($chunk_text =~ /\#\s+(Updated\s+\d?\d-\D\D\D-\d\d\d\d.)\s$/i) {
[24627]	1638	return $1;
	1639	}
[10019]	1640
[24627]	1641	return undef;
[10019]	1642	}
	1643
	1644
	1645	sub is_resource_bundle_chunk_automatically_translated
	1646	{
	1647	# No resource bundle chunks are automatically translated
	1648	return 0;
	1649	}
	1650
	1651
	1652	sub write_translated_resource_bundle
	1653	{
	1654	my $source_file = shift(@_); # Not used
[11321]	1655	my @source_file_lines = @{shift(@_)};
[10019]	1656	my $source_file_key_to_text_mapping = shift(@_);
	1657	my $target_file = shift(@_);
[11321]	1658	my @target_file_lines = @{shift(@_)}; # Not used
[10019]	1659	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1660	my $target_file_key_to_gti_comment_mapping = shift(@_);
[10019]	1661	my $target_language_code = shift(@_); # Not used
[24627]	1662
[10019]	1663	# Build a mapping from chunk key to source file line, and from source file line to chunk key
[11321]	1664	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
[10019]	1665	my %source_file_line_to_key_mapping = ();
	1666	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1667	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[10019]	1668	}
[24627]	1669
[10019]	1670	# Write the new target file
	1671	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1672	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1673	&throw_fatal_error("Could not write target file $target_file_path.");
[10019]	1674	}
[24627]	1675
[10019]	1676	# Model the new target file on the source file, with the target file translations
[10091]	1677	my $source_file_line_number = 0;
[10019]	1678	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1679	# Fill in the gaps before this chunk starts
	1680	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1681	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1682	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1683	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1684	$source_file_line_number++;
	1685	}
	1686	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1687
	1688	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1689	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1690	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1691
	1692	# If no translation exists for this chunk, show this, and move on
	1693	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1694	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	1695	next;
	1696	}
	1697
	1698	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	1699	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1700	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1701	}
	1702	print TARGET_FILE "\n";
[10019]	1703	}
[24627]	1704
[10019]	1705	close(TARGET_FILE);
	1706	}
	1707
	1708
[11321]	1709	# ==========================================================================================
	1710	# GREENSTONE XML FUNCTIONS
	1711
	1712	sub build_key_to_line_mapping_for_greenstone_xml
	1713	{
	1714	my (@file_lines) = @_;
[24627]	1715
[11321]	1716	my %chunk_key_to_line_mapping = ();
	1717	for (my $i = 0; $i < scalar(@file_lines); $i++) {
[24627]	1718	my $line = $file_lines[$i];
	1719	$line =~ s/(\s*)$//; # Remove any nasty whitespace, carriage returns etc.
	1720
	1721	# Line contains a string to translate
	1722	if ($line =~ /^\s<Text id=\"(.?)\">/) {
	1723	my $chunk_key = $1;
	1724	$line =~ s/\s*$//; # Remove any nasty whitespace
	1725	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1726
	1727	# While there is still text of the string to go...
	1728	my $startline = $i;
	1729	while ($line !~ /<\/Text>$/) {
	1730	$i++;
	1731	if ($i == scalar(@file_lines)) {
	1732	&throw_fatal_error("Could not find end of string $chunk_key.");
	1733	}
	1734	$line = $file_lines[$i];
	1735	$line =~ s/\s*$//; # Remove any nasty whitespace
	1736	$line =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
	1737	}
	1738
	1739	# Map from chunk key to line
	1740	if (!defined($chunk_key_to_line_mapping{$chunk_key})) {
	1741	$chunk_key_to_line_mapping{$chunk_key} = $startline . "-" . $i;
	1742	}
	1743	else {
	1744	&throw_fatal_error("Duplicate key $chunk_key.");
	1745	}
[11321]	1746	}
	1747	}
[24627]	1748
[11321]	1749	return %chunk_key_to_line_mapping;
	1750	}
	1751
	1752
	1753	sub import_chunk_from_greenstone_xml
	1754	{
	1755	my ($chunk_text) = @_;
[24627]	1756
[11321]	1757	# Simple: just remove the Text tags
[11447]	1758	$chunk_text =~ s/^\s<Text id=\"(.?)\">(\s*)//;
[12484]	1759	$chunk_text =~ s/<Updated date=\"\d?\d-\D\D\D-\d\d\d\d.*\"\/>$//;
[11321]	1760	$chunk_text =~ s/<\/Text>$//;
[24627]	1761
[11321]	1762	return $chunk_text;
	1763	}
	1764
	1765
[12483]	1766	sub get_greenstone_xml_chunk_gti_comment
[11321]	1767	{
	1768	my ($chunk_text) = @_;
[24627]	1769
[11321]	1770	# Check for an "Updated DD-MMM-YYYY" comment at the end of the chunk
[12484]	1771	if ($chunk_text =~ /<Updated date=\"(\d?\d-\D\D\D-\d\d\d\d.*)\"\/>$/i) {
[24627]	1772	return $1;
[11487]	1773	}
[24627]	1774
[11321]	1775	return undef;
	1776	}
	1777
	1778
	1779	sub is_greenstone_xml_chunk_automatically_translated
	1780	{
	1781	# No greenstone XML chunks are automatically translated
	1782	return 0;
	1783	}
	1784
	1785
	1786	sub write_translated_greenstone_xml
	1787	{
	1788	my $source_file = shift(@_); # Not used
	1789	my @source_file_lines = @{shift(@_)};
	1790	my $source_file_key_to_text_mapping = shift(@_);
	1791	my $target_file = shift(@_);
	1792	my @target_file_lines = @{shift(@_)}; # Not used
	1793	my $target_file_key_to_text_mapping = shift(@_);
[12483]	1794	my $target_file_key_to_gti_comment_mapping = shift(@_);
[11321]	1795	my $target_language_code = shift(@_); # Not used
[24627]	1796
[11321]	1797	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	1798	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_greenstone_xml(@source_file_lines);
	1799	my %source_file_line_to_key_mapping = ();
	1800	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
[24627]	1801	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[11321]	1802	}
[24627]	1803
[11321]	1804	# Write the new target file
	1805	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	1806	if (!open(TARGET_FILE, ">$target_file_path")) {
[24627]	1807	&throw_fatal_error("Could not write target file $target_file_path.");
[11321]	1808	}
[24627]	1809
[11321]	1810	# Model the new target file on the source file, with the target file translations
	1811	my $source_file_line_number = 0;
	1812	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
[24627]	1813	# Fill in the gaps before this chunk starts
	1814	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	1815	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	1816	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	1817	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1818	$source_file_line_number++;
	1819	}
	1820	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	1821
	1822	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	1823	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$chunk_key};
	1824	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$chunk_key} \|\| "";
	1825	$target_file_chunk_text =~ s/(\n)*$//g;
	1826
	1827	# If no translation exists for this chunk, show this, and move on
	1828	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	1829	print TARGET_FILE "<!-- Missing translation: $chunk_key -->\n";
	1830	next;
	1831	}
	1832
	1833	print TARGET_FILE "<Text id=\"$chunk_key\">$target_file_chunk_text</Text>";
	1834	if ($target_file_key_to_gti_comment_mapping->{$chunk_key}) {
	1835	my $chunk_gti_comment = $target_file_key_to_gti_comment_mapping->{$chunk_key};
	1836	$chunk_gti_comment =~ s/^Updated //;
	1837	print TARGET_FILE "<Updated date=\"" . $chunk_gti_comment . "\"\/>";
	1838	}
	1839	print TARGET_FILE "\n";
[11321]	1840	}
[24627]	1841
[11321]	1842	# Fill in the end of the file
	1843	while ($source_file_line_number < scalar(@source_file_lines)) {
[24627]	1844	print TARGET_FILE $source_file_lines[$source_file_line_number];
	1845	$source_file_line_number++;
[11321]	1846	}
[24627]	1847
[11321]	1848	close(TARGET_FILE);
	1849	}
	1850
	1851
[18460]	1852	# ==========================================================================================
	1853	# GREENSTONE3 FUNCTIONS
	1854
	1855	sub get_all_chunks_for_gs3
	1856	{
	1857	# The code of the target language (ensure it is lowercase)
	1858	my $target_language_code = lc(shift(@_));
	1859	my $translation_file_key = lc(shift(@_));
[24627]	1860
[18460]	1861	# Check that the necessary arguments were supplied
	1862	if (!$target_language_code) {
[24627]	1863	&throw_fatal_error("Missing command argument.");
[18460]	1864	}
[24627]	1865
[18460]	1866	# Get (and check) the translation configuration
	1867	# my ($source_file_dir, $target_file, $translation_file_type) = &get_translation_configuration($target_language_code, $translation_file_key);
	1868
	1869	my %source_files_key_to_text_mapping = ();
	1870	my %target_files_key_to_text_mapping = ();
	1871	my %source_files_key_to_last_update_date_mapping = ();
	1872	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1873
[18460]	1874	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[24627]	1875
[18460]	1876	&log_message("Total number of source chunks: " . scalar(keys(%source_files_key_to_text_mapping)));
	1877	&log_message("Total number of target chunks: " . scalar(keys(%target_files_key_to_text_mapping)));
[24627]	1878
[18460]	1879	my $xml_response = &create_xml_response_for_all_chunks($translation_file_key, "", \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1880	return $xml_response;
	1881	}
	1882
	1883
	1884	sub get_first_n_chunks_requiring_work_for_gs3
	1885	{
	1886	# The code of the target language (ensure it is lowercase)
	1887	my $target_language_code = lc(shift(@_));
	1888	# The key of the file to translate (ensure it is lowercase)
	1889	my $translation_file_key = lc(shift(@_));
	1890	# The number of chunks to return (defaults to one if not specified)
	1891	my $num_chunks_to_return = shift(@_) \|\| "1";
	1892
	1893	# Check that the necessary arguments were supplied
	1894	if (!$target_language_code \|\| !$translation_file_key) {
[24627]	1895	&throw_fatal_error("Missing command argument.");
[18460]	1896	}
[28755]	1897
[18460]	1898	my %source_files_key_to_text_mapping = ();
	1899	my %target_files_key_to_text_mapping = ();
	1900	my %source_files_key_to_last_update_date_mapping = ();
	1901	my %target_files_key_to_last_update_date_mapping = ();
[24627]	1902
[18460]	1903	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
[24627]	1904	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
[18460]	1905
	1906	# Determine the target file chunks requiring translation
	1907	my @target_files_keys_requiring_translation = &determine_chunks_requiring_translation(\%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping);
	1908	# Determine the target file chunks requiring updating
	1909	my @target_files_keys_requiring_updating = &determine_chunks_requiring_updating(\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1910	&log_message("Total number of target chunks requiring translation: " . scalar(@target_files_keys_requiring_translation));
	1911	&log_message("Total number of target chunks requiring updating: " . scalar(@target_files_keys_requiring_updating));
[29415]	1912
	1913	my $download_target_filepath = "";
	1914
	1915
	1916	# **** DOWNLOADING LANGUAGE FILES WAS NOT YET IMPLEMENTED FOR GS3. RUDIMENTARY VERSION **** #
	1917
	1918	# if there is no copy of the language files for download, there's also no link to the spreadsheet
	1919	# for translating offline. So GS3's download option, we will zip up all the relevant greenstone 3
	1920	# interface *.properties files,and link to that zip as the file for offline translation.
	1921	# Selecting only properties files for English and the language they're working on (if the last exists)
	1922
	1923	# tar -cvzf gs3interface.tar.gz greenstone3/AbstractBrowse.properties greenstone3/AbstractBrowse_nl.properties
	1924	# will generate a tar file containing a folder called "greenstone3" with the specified *.properties files
	1925
	1926	my $sourcedir = "greenstone3";
	1927	my $zip = &FileUtils::filenameConcatenate("tmp", "gs3interface_".$target_language_code.".tar.gz");
	1928	my $tar_cmd = "tar -cvzf $zip";
	1929
	1930
	1931	# store cur dir and cd to gsdlhome to generate the correct path in the zip file
	1932	my $curdir = `pwd`;
	1933	chdir $gsdl_root_directory;
	1934
	1935	my $filelisting = "";
	1936	foreach my $interface_file (@gs3_interface_files) {
	1937
	1938	my $source_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file.".properties");
	1939	my $target_filepath = &FileUtils::filenameConcatenate($sourcedir, $interface_file."_".$target_language_code.".properties");
[24627]	1940
[29415]	1941	$filelisting = "$filelisting $source_filepath";
	1942	if(&FileUtils::fileExists($target_filepath)) {
	1943	$filelisting = "$filelisting $target_filepath";
[30490]	1944	}
[29415]	1945	}
	1946	$tar_cmd .= " $filelisting";
	1947
	1948	# tar command will overwrite the previous version, but want to check we've created it
	1949	if(&FileUtils::fileExists($zip)) {
	1950	&FileUtils::removeFiles($zip);
	1951	}
	1952
	1953	#my $tar_result = system($tar_cmd); # works but then interface breaks
	1954	`$tar_cmd`;
	1955	my $tar_result = $?;
	1956
	1957	if(&FileUtils::fileExists($zip)) { ## if($tar_result == 0) {, # breaks the interface
	1958	$download_target_filepath = $zip;
	1959	} else {
	1960	&log_message("Unable to generate zip containing gs3interface files " . $download_target_filepath . "$!");
	1961	}
	1962
	1963	# change back to original working directory (cgi-bin/linux probably)
	1964	chdir $curdir;
	1965
	1966	# ************ END RUDIMENTARY VERSION OF DOWNLOADING LANGUAGE FILES FOR GS3 *********** #
	1967
	1968
	1969	my $xml_response = &create_xml_response_for_chunks_requiring_work($translation_file_key, $download_target_filepath, scalar(keys(%source_files_key_to_text_mapping)),
[24627]	1970	\@target_files_keys_requiring_translation, \@target_files_keys_requiring_updating,
	1971	$num_chunks_to_return, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1972	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1973
[18460]	1974	return $xml_response;
	1975	}
	1976
[25249]	1977	sub get_uptodate_chunks_for_gs3
	1978	{
	1979	# The code of the target language (ensure it is lowercase)
	1980	my $target_language_code = lc(shift(@_));
	1981	# The key of the file to translate (ensure it is lowercase)
	1982	my $translation_file_key = lc(shift(@_));
	1983	# The number of chunks to return (defaults to one if not specified)
	1984	my $num_chunks_to_return = shift(@_) \|\| "1";
	1985
	1986	# Check that the necessary arguments were supplied
	1987	if (!$target_language_code \|\| !$translation_file_key) {
	1988	&throw_fatal_error("Missing command argument.");
	1989	}
	1990
	1991	my %source_files_key_to_text_mapping = ();
	1992	my %target_files_key_to_text_mapping = ();
	1993	my %source_files_key_to_last_update_date_mapping = ();
	1994	my %target_files_key_to_last_update_date_mapping = ();
	1995
	1996	&build_gs3_configuration($target_language_code, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping,
	1997	\%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	1998
[18460]	1999
[25249]	2000	# Chunks needing updating are those in the target file that have been more recently edited in the source file
	2001	# All others are uptodate (which implies that they have certainly been translated at some point and would not be empty)
	2002	my @uptodate_target_file_keys = ();
	2003	foreach my $chunk_key (keys(%source_files_key_to_last_update_date_mapping)) {
	2004	my $source_chunk_last_update_date = $source_files_key_to_last_update_date_mapping{$chunk_key};
	2005	my $target_chunk_last_update_date = $target_files_key_to_last_update_date_mapping{$chunk_key};
	2006
	2007	# print "key: $chunk_key\nsource date : $source_chunk_last_update_date\ntarget date : $target_chunk_last_update_date\nafter? ". &is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date) . "\n\n";
	2008
	2009	if (defined($target_chunk_last_update_date) && !&is_date_after($source_chunk_last_update_date, $target_chunk_last_update_date)) {
	2010	# &log_message("Chunk with key $chunk_key needs updating.");
	2011	push(@uptodate_target_file_keys, $chunk_key);
	2012	}
	2013	}
[18460]	2014
[25249]	2015	my $xml_response = &create_xml_response_for_uptodate_chunks($translation_file_key, "", \@uptodate_target_file_keys, \%source_files_key_to_text_mapping, \%target_files_key_to_text_mapping, \%source_files_key_to_last_update_date_mapping, \%target_files_key_to_last_update_date_mapping);
	2016
	2017	return $xml_response;
	2018	}
	2019
	2020
[18460]	2021	sub build_gs3_configuration
	2022	{
	2023	my ($target_language_code, $source_files_key_to_text_mapping, $target_files_key_to_text_mapping,
	2024	$source_files_key_to_gti_comment_mapping, $target_files_key_to_gti_comment_mapping) = @_;
	2025
[28755]	2026	my $source_file_directory = "greenstone3"; # my $source_file_directory = &util::filename_cat("WEB-INF","classes");
[18460]	2027	my $translation_file_type = "resource_bundle";
[24627]	2028
[18460]	2029	foreach my $interface_file_key (@gs3_interface_files) {
[24627]	2030
	2031	&log_message("Greenstone 3 interface file: " . $interface_file_key);
	2032
	2033	# Parse the source language and target language files
	2034	my $source_file = &util::filename_cat($source_file_directory, $interface_file_key.".properties");
	2035	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	2036	my %source_file_key_to_line_mapping = &build_key_to_line_mapping(\@source_file_lines, $translation_file_type);
	2037	my %source_file_key_to_text_mapping = &build_key_to_text_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	2038	my %source_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@source_file_lines, \%source_file_key_to_line_mapping, $translation_file_type);
	2039
	2040	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key."_".$target_language_code.".properties");
	2041	my @target_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $target_file));
	2042	my %target_file_key_to_line_mapping = &build_key_to_line_mapping(\@target_file_lines, $translation_file_type);
	2043	my %target_file_key_to_text_mapping = &build_key_to_text_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	2044	my %target_file_key_to_gti_comment_mapping = &build_key_to_gti_comment_mapping(\@target_file_lines, \%target_file_key_to_line_mapping, $translation_file_type);
	2045
	2046
	2047	# Filter out any automatically translated chunks
	2048	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	2049	if (&is_chunk_automatically_translated($chunk_key, $translation_file_type)) {
	2050	delete $source_file_key_to_line_mapping{$chunk_key};
	2051	delete $target_file_key_to_line_mapping{$chunk_key};
	2052	}
	2053	}
	2054
	2055	&log_message("Number of source chunks: " . scalar(keys(%source_file_key_to_text_mapping)));
	2056	&log_message("Number of target chunks: " . scalar(keys(%target_file_key_to_text_mapping)));
	2057
	2058	foreach my $chunk_key (keys(%source_file_key_to_text_mapping)) {
	2059	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2060	$source_files_key_to_text_mapping->{$global_chunk_key} = $source_file_key_to_text_mapping{$chunk_key};
	2061	$source_files_key_to_gti_comment_mapping->{$global_chunk_key} = $source_file_key_to_gti_comment_mapping{$chunk_key};
	2062
	2063	if (defined $target_file_key_to_text_mapping{$chunk_key}) {
	2064	$target_files_key_to_text_mapping->{$global_chunk_key} = $target_file_key_to_text_mapping{$chunk_key};
	2065	$target_files_key_to_gti_comment_mapping->{$global_chunk_key} = $target_file_key_to_gti_comment_mapping{$chunk_key};
	2066	}
[29456]	2067	}
[18460]	2068	}
	2069	}
	2070
	2071
	2072	sub write_translated_gs3interface
	2073	{
	2074	my $source_file_key_to_text_mapping = shift(@_);
	2075	my $target_file_key_to_text_mapping = shift(@_);
	2076	my $target_file_key_to_gti_comment_mapping = shift(@_);
	2077	my $target_language_code = shift(@_);
	2078
	2079	my @sorted_chunk_keys = sort (keys(%$source_file_key_to_text_mapping));
[24627]	2080
[18460]	2081	my %translated_interface_file_keys = ();
	2082	foreach my $chunk_key (keys(%$target_file_key_to_text_mapping)) {
[24627]	2083	$chunk_key =~ /^([^\.]+)?\.(.*)$/;
	2084	if (!defined $translated_interface_file_keys{$1}) {
	2085	&log_message("Updated interface file: " . $1);
	2086	$translated_interface_file_keys{$1}="";
	2087	}
[18460]	2088	}
	2089	&log_message("Updated interface files: " . scalar(keys(%translated_interface_file_keys)));
	2090
	2091	my $source_file_directory = "greenstone3";
	2092
	2093	foreach my $interface_file_key (keys(%translated_interface_file_keys)) {
[24627]	2094
	2095	# Build a mapping from chunk key to source file line, and from source file line to chunk key
	2096	my $source_file = &util::filename_cat($source_file_directory, "$interface_file_key.properties");
	2097	my @source_file_lines = &read_file_lines(&util::filename_cat($gsdl_root_directory, $source_file));
	2098	my %source_file_key_to_line_mapping = &build_key_to_line_mapping_for_resource_bundle(@source_file_lines);
	2099	my %source_file_line_to_key_mapping = ();
	2100	foreach my $chunk_key (keys(%source_file_key_to_line_mapping)) {
	2101	$source_file_line_to_key_mapping{$source_file_key_to_line_mapping{$chunk_key}} = $chunk_key;
[18460]	2102	}
[24627]	2103
	2104	# Write the new target file
	2105	my $target_file = &util::filename_cat($source_file_directory, $interface_file_key . "_" . $target_language_code . ".properties");
	2106	my $target_file_path = &util::filename_cat($gsdl_root_directory, $target_file);
	2107	if (!open(TARGET_FILE, ">$target_file_path")) {
	2108	&throw_fatal_error("Could not write target file $target_file_path.");
	2109	}
	2110
	2111	# Model the new target file on the source file, with the target file translations
	2112	my $source_file_line_number = 0;
	2113	foreach my $line_key (sort sort_by_line (keys(%source_file_line_to_key_mapping))) {
	2114	# Fill in the gaps before this chunk starts
	2115	my $source_file_chunk_starting_line_number = (split(/-/, $line_key))[0];
	2116	my $source_file_chunk_finishing_line_number = (split(/-/, $line_key))[1];
	2117	while ($source_file_line_number < $source_file_chunk_starting_line_number) {
	2118	print TARGET_FILE $source_file_lines[$source_file_line_number];
	2119	$source_file_line_number++;
	2120	}
	2121	$source_file_line_number = $source_file_chunk_finishing_line_number + 1;
	2122
	2123	my $chunk_key = $source_file_line_to_key_mapping{$line_key};
	2124	my $global_chunk_key = "$interface_file_key.$chunk_key";
	2125	my $source_file_chunk_text = $source_file_key_to_text_mapping->{$global_chunk_key};
	2126	my $target_file_chunk_text = $target_file_key_to_text_mapping->{$global_chunk_key} \|\| "";
	2127
	2128	# If no translation exists for this chunk, show this, and move on
	2129	if ($source_file_chunk_text ne "" && $target_file_chunk_text eq "") {
	2130	print TARGET_FILE "# -- Missing translation: $chunk_key\n";
	2131	next;
	2132	}
	2133
	2134	print TARGET_FILE "$chunk_key:$target_file_chunk_text";
	2135	if ($target_file_key_to_gti_comment_mapping->{$global_chunk_key}) {
	2136	print TARGET_FILE " # " . $target_file_key_to_gti_comment_mapping->{$global_chunk_key};
	2137	}
	2138	print TARGET_FILE "\n";
	2139	}
	2140
	2141	close(TARGET_FILE);
[18460]	2142	}
	2143	}
	2144
[10019]	2145	&main(@ARGV);

Note: See TracBrowser for help on using the repository browser.

Download in other formats: