Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ArchivesInfPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ArchivesInfPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,330 +1,0 @@
-###########################################################################
-#
-# ArchivesInfPlugin.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# plugin which reads through an archives.inf (or archiveinf-doc info database equivalent)
-# -- i.e. the file generated in the archives directory when an import is done),
-# processing each file it finds
-
-package ArchivesInfPlugin;
-
-use util;
-use doc;
-use PrintInfo;
-use plugin;
-use arcinfo;
-use gsprintf;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-
-BEGIN {
-    @ArchivesInfPlugin::ISA = ('PrintInfo');
-}
-
-my $arguments = [
-      { 'name' => "reversesort",
-	'desc' => "{ArchivesInfPlugin.reversesort}",
-	'type' => "flag",
-	'reqd' => "no",
-	'modegli' => "2" },
-
-		 ];
-
-my $options = { 'name'     => "ArchivesInfPlugin",
-		'desc'     => "{ArchivesInfPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-	        'args' => $arguments};
-         
-sub gsprintf
-{
-    return &gsprintf::gsprintf(@_);
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-# called once, at the start of processing
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    # verbosity is passed through from the processor
-    $self->{'verbosity'} = $verbosity;
-
-    # as are the outhandle and failhandle
-    $self->{'outhandle'} = $outhandle if defined $outhandle;
-    $self->{'failhandle'} = $failhandle;
-
-}
-
-sub deinit {
-    my ($self) = @_;
-
-    my $archive_info = $self->{'archive_info'};
-    my $verbosity = $self->{'verbosity'};
-    my $outhandle = $self->{'outhandle'};
-
-    if (defined $archive_info) {
-        # Get the infodbtype value for this collection from the arcinfo object
-        my $infodbtype = $archive_info->{'infodbtype'};
-	my $archive_info_filename = $self->{'archive_info_filename'};
-	my $infodb_file_handle = &dbutil::open_infodb_write_handle($infodbtype, $archive_info_filename, "append");
-
-       	my $file_list = $archive_info->get_file_list();
-
-	foreach my $subfile (@$file_list) {	    
-	    my $doc_oid = $subfile->[1];
-
-	    my $index_status = $archive_info->get_status_info($doc_oid);
-
-	    if ($index_status eq "D") {
-		# delete
-		$archive_info->delete_info($doc_oid);
-		&dbutil::delete_infodb_entry($infodbtype, $infodb_file_handle, $doc_oid);
-
-		my $doc_file = $subfile->[0];
-		my $base_dir =$self->{'base_dir'};
-
-		my $doc_filename = &util::filename_cat($base_dir,$doc_file);
-
-		my ($doc_tailname, $doc_dirname, $suffix) 
-		    = File::Basename::fileparse($doc_filename, "\\.[^\\.]+\$");
-
-		print $outhandle "Removing $doc_dirname\n" if ($verbosity>2);
-
-		&util::rm_r($doc_dirname);
-
-
-	    }
-	    elsif ($index_status =~ m/^(I|R)$/) {
-		# mark as "been indexed"
-		$archive_info->set_status_info($doc_oid,"B");
-	    }
-	}
-
-	&dbutil::close_infodb_write_handle($infodbtype, $infodb_file_handle);
-	$archive_info->save_info($archive_info_filename);
-    }
-}
-
-# called at the beginning of each plugin pass (import has one, buildin has many)
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    $self->{'base_dir'} = $base_dir;
-}
-
-sub remove_all {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-}
-
-sub remove_one {
-    my $self = shift (@_);
-    my ($file, $oids, $archivedir) = @_;
-    return undef; # only called during import at this stage, this will never be processing a file
-    
-}
-
-
-# called at the end of each plugin pass
-sub end {
-    my ($self) = shift (@_);
-
-}
-
-
-# return 1 if this class might recurse using $pluginfo
-sub is_recursive {
-    my $self = shift (@_);
-
-    return 1;
-}
-
-
-sub compile_stats {
-    my $self = shift(@_);
-    my ($stats) = @_;
-}
-
-# We don't do metadata_read
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    return undef;
-}
-
-sub file_block_read {
-
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
-
-    if ($file eq "OIDcount") {
-	my ($filename_full_path, $filename_no_path) 
-	    = &util::get_full_filenames($base_dir, $file);
-	&util::block_filename($block_hash,$filename_full_path);
-	return 1;
-    }
-
-    # otherwise, we don't do any file blocking
-
-    return undef;
-}
-
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs,$total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $count = 0;
-
-    # This function only makes sense at build-time
-    return if (ref($processor) !~ /buildproc$/i);
-
-    # Get the infodbtype value for this collection from the buildproc object
-    my $infodbtype = $processor->{'infodbtype'};
-    $infodbtype = "gdbm" if $infodbtype eq "gdbm-txtgz";
-    
-    # see if this has a archives information file within it
-##    my $archive_info_filename = &util::filename_cat($base_dir,$file,"archives.inf");
-    my $archive_info_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", &util::filename_cat($base_dir, $file));
-
-    if (-e $archive_info_filename) {
-
-	# found an archives.inf file
-	&gsprintf($outhandle, "ArchivesInfPlugin: {common.processing} $archive_info_filename\n") if $self->{'verbosity'} > 1;
-
-	# read in the archives information file
-	my $archive_info = new arcinfo($infodbtype);
-	$self->{'archive_info'} = $archive_info;
-	$self->{'archive_info_filename'} = $archive_info_filename;
-	if ($self->{'reversesort'}) {
-	    $archive_info->reverse_sort();
-	}
-	
-	$archive_info->load_info ($archive_info_filename);
-	
-	my $file_list = $archive_info->get_file_list();
-
-	# process each file
-	foreach my $subfile (@$file_list) {
-
-	    last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs);
-
-	    my $tmp = &util::filename_cat ($file, $subfile->[0]);
-	    next if $tmp eq $file;
-
-	    my $doc_oid = $subfile->[1];
-	    my $index_status = $archive_info->get_status_info($doc_oid);
-
-	    my $curr_mode = $processor->get_mode();
-	    my $new_mode = $curr_mode;
-
-	    # Start by assuming we want to process the file...
-	    my $process_file = 1;
-
-	    # ...unless the build processor is incremental capable and -incremental was specified, in which case we need to check its index_status flag
-	    if ($processor->is_incremental_capable() && $self->{'incremental'})
-	    {
-	        # Check to see if the file needs indexing
-		if ($index_status eq "B")
-		{
-		    # Don't process this file as it has already been indexed
-		    $process_file = 0;
-		}
-		elsif ($index_status eq "D") {
-		    # Need to be delete it from the index.
-		    $new_mode = $curr_mode."delete";
-		    $process_file = 1;
-		}
-		elsif ($index_status eq "R") {
-		    # Need to be reindexed/replaced
-		    $new_mode = $curr_mode."reindex";
-
-		    $process_file = 1;
-		}
-	    }
-	    # ... or we're being asked to delete it (in which case skip it)
-	    elsif ($index_status eq "D") {
-		# Non-incremental Delete
-		# It's already been deleted from the archives directory
-		# (done during import.pl)
-		# => All we need to do here is not process it
-
-		$process_file = 0;
-	    }
-
-	    if (!$processor->is_incremental_capable() && $self->{'incremental'}) {
-		# Nag feature
-		if (!defined $self->{'incremental-warning'}) {
-		    print $outhandle "\n";
-		    print $outhandle "Warning: command-line option '-incremental' used with *non-incremental*\n";
-		    print $outhandle "         processor '", ref $processor, "'. Some conflicts may arise.\n";
-		    print $outhandle "\n";
-		    sleep 10;
-		    $self->{'incremental-warning'} = 1;
-		}
-	    }
-
-	    if ($process_file) {
-		# note: metadata is not carried on to the next level
-		
-		$processor->set_mode($new_mode) if ($new_mode ne $curr_mode);
-
-		$count += &plugin::read ($pluginfo, $base_dir, $tmp, $block_hash, {}, $processor, $maxdocs, ($total_count+$count), $gli);
-
-		$processor->set_mode($curr_mode) if ($new_mode ne $curr_mode);
-	    }
-	}
-
-	return $count;
-    }
-
-
-    # wasn't an archives directory, someone else will have to process it
-    return undef;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/AutoLoadConverters.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/AutoLoadConverters.pm	(revision 26941)
+++ 	(revision )
@@ -1,174 +1,0 @@
-package AutoLoadConverters;
-
-use strict;
-no strict 'refs';
-no strict 'subs';
-
-use PrintInfo;
-
-use gsprintf 'gsprintf';
-
-sub BEGIN {
-    @AutoLoadConverters::ISA = ('PrintInfo');
-}
-
-# the dynamic conversion args will get added here
-my $arguments = [];
-
-my $options = { 'name'     => "AutoLoadConverters",
-		'desc'     => "{AutoLoadConverters.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub dynamically_load_converter
-{
-    my ($autoPackage, $calling_class) = @_;
-
-    my $settings = {};
-    
-    my ($autoName)  = ($autoPackage =~ m/(^.*)Converter$/);
-    my $autoVar = lc($autoName);
-    $settings->{'converter_name'} = $autoName;
-    $settings->{'converter_var'} = $autoVar;
-    
-    eval("require $autoPackage");
-    if ($@) {
-	# Useful debugging statement if there is a syntax error in the plugin
-	#print STDERR "$@\n";
-	$settings->{'converter_installed'} = 0;
-	$settings->{'conversion_available'} = 0;
-    }
-    else {
-	# found the converter
-	$settings->{'conversion_installed'} = 1;
-	print STDERR "AutoLoadConverters: $autoName Extension to Greenstone detected for $calling_class\n";
-	# but can it run???
-	if (eval "\$${autoPackage}::${autoVar}_conversion_available") {
-	    $settings->{'conversion_available'} = 1;
-	    push(@AutoLoadConverters::ISA, $autoPackage);
-
-	} else {
-	    $settings->{'conversion_available'} = 0;
-	    print STDERR "... but it appears to be broken\n";
-	    &gsprintf(STDERR, "AutoLoadConverters: {AutoloadConverter.noconversionavailable}");
-	    my $dictentry_reason
-		= eval "$autoPackage\.\$${autoPackage}::no_${autoVar}_conversion_reason";
-
-	    &gsprintf(STDERR, " ({$dictentry_reason})\n");
-
-	}
-    }
-
-    if ($settings->{'conversion_available'}) {
-
-	my $opt_conversion_args = 
-	    [ { 'name' => "$autoVar\_conversion",
-		'desc' => "{$autoPackage.$autoVar\_conversion}",
-		'type' => "flag",
-		'reqd' => "no" } ];
-
-	$settings->{'converter_arguments'} = $opt_conversion_args;
-
-    }
-    return $settings;
-
-
-}
-
-####
-# This plugin takes an extra initial parameter in its constructor (compared 
-#   with the norm).  The extra parameter is an array of converters 
-# it should  try to dynamically load
-#####
-
-sub new {
-
-    # Start the AutoExtractMetadata Constructor
-    my $class = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists,$autoPackages, $auxiliary) = @_;
-    
-    push(@$pluginlist, $class);
-    my $classPluginName = $pluginlist->[0];
-
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    # this static list is just for pluginfo on AutoLoadConverters - we will try to explicitly load all packages 
-    $autoPackages = ["OpenOfficeConverter", "PDFBoxConverter"] unless defined $autoPackages;
-
-    my @self_array = ();
-    my $temporary_self = {};
-    $temporary_self->{'converter_list'} = [];
-    push (@self_array, $temporary_self);
-    foreach my $packageName (@$autoPackages) {
-	my $package_settings = &dynamically_load_converter($packageName, $classPluginName);
-	my $available_var = $package_settings->{'converter_var'}."_available";
-	if ($package_settings->{'conversion_available'}) {
-	    push(@$arguments, @{$package_settings->{'converter_arguments'}});
-	    my $package_self = eval "new $packageName(\$pluginlist, \$inputargs, \$hashArgOptLists,1);";
-	    push (@self_array, $package_self);
-	    $temporary_self->{$available_var} = 1;
-	    push(@{$temporary_self->{'converter_list'}}, $packageName);
-	} else {
-	    $temporary_self->{$available_var} = 0;
-	}
-    }
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    my $self;
-    my $pi_self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists, 1);
-    push (@self_array, $pi_self);
-    $self = BasePlugin::merge_inheritance(@self_array);
-
-    return bless $self, $class;
-}
-
-sub init {
-    my $self = shift (@_);
-
-    foreach my $converter (@{$self->{'converter_list'}}) {
-	eval "\$self->${converter}::init(\@_);";
-    }
-
-}
-
-sub begin {
-    my $self = shift (@_);
-
-    foreach my $converter(@{$self->{'converter_list'}}) {
-	eval "\$self->${converter}::begin(\@_);";
-    }
-}
-
-sub deinit {
-    my $self = shift (@_);
-
-    foreach my $converter (@{$self->{'converter_list'}}) {
-	eval "\$self->${converter}::deinit(\@_);";
-    }
-
-}
-
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename, $textref) = @_;
-
-    foreach my $converter(@{$self->{'converter_list'}}) {
-	my ($var) = ($converter =~ m/(^.*)Converter$/);
-	$var = lc($var);
-	if ($self->{"${var}_conversion"}) {
-	    my ($result, $result_str, $new_filename) 
-		= eval "\$self->${converter}::convert(\$input_filename, \$output_ext);";
-	    if ($result != 0) {
-		return $new_filename;
-	    }
-	    my $outhandle=$self->{'outhandle'};
-	    print $outhandle "$converter Conversion error\n";
-	    print $outhandle $result_str;
-	    return "";
-	}
-    }
-    
-    # if got here, no converter was specified in plugin args
-    return $self->ConvertBinaryFile::tmp_area_convert_file(@_);
-    
-}
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/EmailAddressExtractor.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/EmailAddressExtractor.pm	(revision 26941)
+++ 	(revision )
@@ -1,129 +1,0 @@
-###########################################################################
-#
-# EmailAddressExtractor - helper plugin that extracts email addresses from text
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package EmailAddressExtractor;
-
-use PrintInfo;
-use strict;
-
-BEGIN {
-    @EmailAddressExtractor::ISA = ('PrintInfo');
-}
-
-my $arguments = [
-      { 'name' => "extract_email",
-	'desc' => "{EmailAddressExtractor.extract_email}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "new_extract_email",
-	'desc' => "",
-	'type' => "flag",
-	'reqd' => "no",
-	'hiddengli' => "yes" } 
-		 ];
-
-my $options = { 'name'     => "EmailAddressExtractor",
-		'desc'     => "{EmailAddressExtractor.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists, 1);
-
-    return bless $self, $class;
-
-}
-
-# extract metadata
-sub extract_email_metadata {
-
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-    
-    if ($self->{'extract_email'}) {
-	my $thissection = $doc_obj->get_top_section();
-	while (defined $thissection) {
-	    my $text = $doc_obj->get_text($thissection);
-	    $self->extract_email (\$text, $doc_obj, $thissection) if $text =~ /./;
-	    $thissection = $doc_obj->get_next_section ($thissection);
-	}
-    } 
-
-}
-
-sub extract_email {
-    my $self = shift (@_);
-    my ($textref, $doc_obj, $thissection) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    gsprintf($outhandle, " {EmailAddressExtractor.extracting_emails}...\n")
-	if ($self->{'verbosity'} > 2);
-    
-    my @email = ($$textref =~ m/([-a-z0-9\.@+_=]+@(?:[-a-z0-9]+\.)+(?:com|org|edu|mil|int|net|[a-z][a-z]))/g);
-    @email = sort @email;
-    
-#    if($self->{"new_extract_email"} == 0)
-#    {
-#    my @email2 = ();
-#    foreach my $address (@email) 
-#   {
-#	if (!(join(" ",@email2) =~ m/(^| )$address( |$)/ )) 
-#	    {
-#		push @email2, $address;
-#		$doc_obj->add_utf8_metadata ($thissection, "EmailAddress", $address);
-#		# print $outhandle "  extracting $address\n" 
-#		&gsprintf($outhandle, "  {AutoExtractMetadata.extracting} $address\n")
-#		    if ($self->{'verbosity'} > 3);
-#	    }
-#	}
-#    }
-#    else
-#    {
-    my $hashExistMail = {};
-    foreach my $address (@email) {
-	if (!(defined $hashExistMail->{$address}))
-	{
-	    $hashExistMail->{$address} = 1;
-	    $doc_obj->add_utf8_metadata ($thissection, "EmailAddress", $address);
-	    gsprintf($outhandle, "  {AutoExtractMetadata.extracting} $address\n")
-		if ($self->{'verbosity'} > 3);
-	}
-    }
-    gsprintf($outhandle, " {EmailAddressExtractor.done_email_extract}\n")
-	if ($self->{'verbosity'} > 2);
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/FavouritesPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/FavouritesPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,111 +1,0 @@
-###########################################################################
-#
-# FavouritesPlug.pm -- Plugin for Internet Explorer Favourites files
-#                      By Stephen De Gabrielle
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-# hacked together by Stephen De Gabrielle from various plugins 
-# especially SRCPlug by John McPherson Nov 2000
-
-package FavouritesPlugin;
-
-use ReadTextFile;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @FavouritesPlugin::ISA = ('ReadTextFile');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "FavouritesPlugin",
-		'desc'     => "{FavouritesPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift(@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-sub get_default_process_exp
-{
-    # URL is extension for single bookmarks under windows.
-    return q^(?i)\.URL$^;
-}
-
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $section = $doc_obj->get_top_section();
-
-    # don't want mg to turn escape chars into actual values
-    $$textref =~ s/\\/\\\\/g;
-
-    # use filename (minus the .url extension) as the title
-    my $title = $file;
-    $title =~ s/.url$//i;
-    $doc_obj->add_utf8_metadata($section, "Title", $title);
-
-    # get the URL from the file
-    my ($url) = ($$textref =~ m/^URL=(http.+)/mg);
-    $doc_obj->add_metadata($section, "URL", $url);
-
-    # Add weblink metadata for an automatic link to the webpage
-    $doc_obj->add_utf8_metadata($section, "weblink", "<a href=\"$url\">");
-    $doc_obj->add_utf8_metadata($section, "webicon", "_iconworld_");
-    $doc_obj->add_utf8_metadata($section, "/weblink", "</a>");
-
-    # Tidy up the favourite text to look a bit nicer
-    $$textref =~ s/^\\n/<p>/g;
-    $$textref =~ s/\[/<p><strong>/g;
-    $$textref =~ s/\]/<\/strong><p>/g;
-    $$textref =~ s/^Modified=(.+)$/<strong>Modified<\/strong>$1<p>/g;
-    $doc_obj->add_utf8_text($section, "$$textref");
-
-    $doc_obj->add_metadata($section, "FileFormat", "Favourite");
-    return 1;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/HTMLPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/HTMLPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,1858 +1,0 @@
-###########################################################################
-#
-# HTMLPlugin.pm -- basic html plugin
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# 
-# Note that this plugin handles frames only in a very simple way
-# i.e. each frame is treated as a separate document. This means
-# search results will contain links to individual frames rather
-# than linking to the top level frameset.
-# There may also be some problems caused by the _parent target
-# (it's removed by this plugin)
-#
-
-package HTMLPlugin;
-
-use Encode;
-use Unicode::Normalize 'normalize';
-
-use ReadTextFile;
-use HBPlugin;
-use ghtml;
-use unicode;
-use util;
-use XMLParser;
-
-use Image::Size;
-use File::Copy;
-
-sub BEGIN {
-    @HTMLPlugin::ISA = ('ReadTextFile', 'HBPlugin');
-}
-
-use strict; # every perl program should have this!
-no strict 'refs'; # make an exception so we can use variables as filehandles
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' =>  &get_default_process_exp() },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => 'regexp',
-	'deft' =>  &get_default_block_exp() },
-      { 'name' => "nolinks",
-	'desc' => "{HTMLPlugin.nolinks}",
-	'type' => "flag" },
-      { 'name' => "keep_head",
-	'desc' => "{HTMLPlugin.keep_head}",
-	'type' => "flag" },
-      { 'name' => "no_metadata",
-	'desc' => "{HTMLPlugin.no_metadata}",
-	'type' => "flag" },
-      { 'name' => "metadata_fields",
-	'desc' => "{HTMLPlugin.metadata_fields}",
-	'type' => "string",
-	'deft' => "Title" },
-      { 'name' => "metadata_field_separator",
-	'desc' => "{HTMLPlugin.metadata_field_separator}",
-	'type' => "string",
-	'deft' => "" },
-      { 'name' => "hunt_creator_metadata",
-	'desc' => "{HTMLPlugin.hunt_creator_metadata}",
-	'type' => "flag" },
-      { 'name' => "file_is_url",
-	'desc' => "{HTMLPlugin.file_is_url}",
-	'type' => "flag" },
-      { 'name' => "assoc_files",
-	'desc' => "{HTMLPlugin.assoc_files}",
-	'type' => "regexp",
-	'deft' => &get_default_block_exp() },
-      { 'name' => "rename_assoc_files",
-	'desc' => "{HTMLPlugin.rename_assoc_files}",
-	'type' => "flag" },
-      { 'name' => "title_sub",
-	'desc' => "{HTMLPlugin.title_sub}",
-	'type' => "string", 
-	'deft' => "" },
-      { 'name' => "description_tags",
-	'desc' => "{HTMLPlugin.description_tags}",
-	'type' => "flag" },
-      # retain this for backward compatibility (w3mir option was replaced by
-      # file_is_url)
-      { 'name' => "w3mir",
-#	'desc' => "{HTMLPlugin.w3mir}",
-	'type' => "flag",
-	'hiddengli' => "yes"},
-      { 'name' => "no_strip_metadata_html",
-	'desc' => "{HTMLPlugin.no_strip_metadata_html}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no"},
-      { 'name' => "sectionalise_using_h_tags",
-	'desc' => "{HTMLPlugin.sectionalise_using_h_tags}",
-	'type' => "flag" },
-      { 'name' => "use_realistic_book",
-        'desc' => "{HTMLPlugin.tidy_html}",
-	'type' => "flag"},
-      { 'name' => "old_style_HDL",
-        'desc' => "{HTMLPlugin.old_style_HDL}",
-	'type' => "flag"},
-      {'name' => "processing_tmp_files",
-       'desc' => "{BasePlugin.processing_tmp_files}",
-       'type' => "flag",
-       'hiddengli' => "yes"}
-      ];
-
-my $options = { 'name'     => "HTMLPlugin",
-		'desc'     => "{HTMLPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-
-    my $self = new ReadTextFile($pluginlist,$inputargs,$hashArgOptLists);
-    
-    if ($self->{'w3mir'}) {
-	$self->{'file_is_url'} = 1;
-    }
-    $self->{'aux_files'} = {};
-    $self->{'dir_num'} = 0;
-    $self->{'file_num'} = 0;
-    
-    return bless $self, $class;
-}
-
-# may want to use (?i)\.(gif|jpe?g|jpe|png|css|js(?:@.*)?)$
-# if have eg <script language="javascript" src="img/lib.js@123">
-# blocking is now done by reading through the file and recording all the 
-# images and other files
-sub get_default_block_exp {
-    my $self = shift (@_);
-    
-    #return q^(?i)\.(gif|jpe?g|jpe|jpg|png|css)$^;
-    return "";
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-    
-    # the last option is an attempt to encode the concept of an html query ...
-    return q^(?i)(\.html?|\.shtml|\.shm|\.asp|\.php\d?|\.cgi|.+\?.+=.*)$^;
-}
-
-sub store_block_files
-{
-    my $self =shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-
-    my $html_fname = $filename_full_path;
-    
-    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
-    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
-
-    # read in file ($text will be in utf8)
-    my $raw_text = "";
-    $self->read_file_no_decoding($filename_full_path, \$raw_text);
-
-    my $textref = \$raw_text;
-    my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
-    my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
-    $$textref =~ s/$opencom(.*?)$closecom//gs;
-
-    # Convert entities to their UTF8 equivalents
-    $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
-    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,0)/gseo; # on this occassion, want it left as utf8
-    $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;
-
-    my $attval = "\\\"[^\\\"]+\\\"|[^\\s>]+";
-    my @img_matches = ($$textref =~ m/<img[^>]*?src\s*=\s*($attval)[^>]*>/igs);
-    my @usemap_matches = ($$textref =~ m/<img[^>]*?usemap\s*=\s*($attval)[^>]*>/igs);
-    my @link_matches = ($$textref =~ m/<link[^>]*?href\s*=\s*($attval)[^>]*>/igs);
-    my @embed_matches = ($$textref =~ m/<embed[^>]*?src\s*=\s*($attval)[^>]*>/igs);
-    my @tabbg_matches = ($$textref =~ m/<(?:body|table|tr|td)[^>]*?background\s*=\s*($attval)[^>]*>/igs);
-    my @script_matches = ($$textref =~ m/<script[^>]*?src\s*=\s*($attval)[^>]*>/igs);
-
-    if(!defined $self->{'unicode_to_original_filename'}) { 
-	# maps from utf8 converted link name -> original filename referrred to by (possibly URL-encoded) src url
-	$self->{'unicode_to_original_filename'} = {};
-    }
-
-    foreach my $raw_link (@img_matches, @usemap_matches, @link_matches, @embed_matches, @tabbg_matches, @script_matches) {
-
-	# remove quotes from link at start and end if necessary
-	if ($raw_link =~ m/^\"/) {
-	    $raw_link =~ s/^\"//;
-	    $raw_link =~ s/\"$//;
-	}
-
-	# remove any anchor names, e.g. foo.html#name becomes foo.html 
-	# but watch out for any #'s that are part of entities, such as &#x3B1;
-	$raw_link =~ s/([^&])\#.*$/$1/s; 
-
-	# some links may just be anchor names
-	next unless ($raw_link =~ /\S+/);
-
-	if ($raw_link !~ m@^/@ && $raw_link !~ m/^([A-Z]:?)\\/i) {
-	    # Turn relative file path into full path
-	    my $dirname = &File::Basename::dirname($filename_full_path);
-	    $raw_link = &util::filename_cat($dirname, $raw_link);
-	}
-	$raw_link = $self->eval_dir_dots($raw_link);
-
-	# this is the actual filename on the filesystem (that the link refers to)
-	my $url_original_filename = $self->opt_url_decode($raw_link);
-
-	my ($uses_bytecodes,$exceeds_bytecodes) = &unicode::analyze_raw_string($url_original_filename);
-
-	if ($exceeds_bytecodes) {
-	    # We have a link to a file name that is more complicated than a raw byte filename
-	    # What we do next depends on the operating system we are on
-
-	    if ($ENV{'GSDLOS'} =~ /^(linux|solaris)$/i) {
-		# Assume we're dealing with a UTF-8 encoded filename
-		$url_original_filename = encode("utf8", $url_original_filename);
-	    }
-	    elsif ($ENV{'GSDLOS'} =~ /^darwin$/i) {
-		# HFS+ is UTF8 with decompostion
-		$url_original_filename = encode("utf8", $url_original_filename);
-		$url_original_filename = normalize('D', $url_original_filename); # Normalization Form D (decomposition)		
-	    }
-	    elsif ($ENV{'GSDLOS'} =~ /^windows$/i) {
-		# Don't need to do anything as later code maps Windows
-		# unicode filenames to DOS short filenames when needed		
-	    }
-	    else {
-		my $outhandle = $self->{'outhandle'};
-		print $outhandle "Warning: Unrecognized operating system ", $ENV{'GSDLOS'}, "\n";
-		print $outhandle "         in raw file system encoding of: $raw_link\n";
-		print $outhandle "         Assuming filesystem is UTF-8 based.\n";
-		$url_original_filename = encode("utf8", $url_original_filename);
-	    }
-	}
-
-	# Convert the (currently raw) link into its Unicode version. 
-	# Store the Unicode link along with the url_original_filename
-	my $unicode_url_original_filename = "";
-	$self->decode_text($raw_link,$content_encoding,$language,\$unicode_url_original_filename);
-
-
-	$self->{'unicode_to_original_filename'}->{$unicode_url_original_filename} = $url_original_filename;
-
-
-	if ($url_original_filename ne $unicode_url_original_filename) {
-	    my $outhandle = $self->{'outhandle'};
-	    
-	    print $outhandle "URL Encoding $url_original_filename\n";
-	    print $outhandle " ->$unicode_url_original_filename\n";
-
-	    # Allow for possibility of raw byte version and Unicode versions of file
-	    &util::block_filename($block_hash,$unicode_url_original_filename);
-	}
-
-	# $url_original_filename = &util::upgrade_if_dos_filename($url_original_filename);
-	&util::block_filename($block_hash,$url_original_filename);
-		
-    }
-}
-
-# Given a filename in any encoding, will URL decode it to get back the original filename
-# in the original encoding. Because this method is intended to work out the *original* 
-# filename*, it does not URL decode any filename if a file by the name of the *URL-encoded*
-# string already exists in the local folder.
-#
-sub opt_url_decode {
-    my $self = shift (@_);
-    my ($raw_link) = @_;
-
-
-    # Replace %XX's in URL with decoded value if required.
-    # Note that the filename may include the %XX in some situations
-
-##    if ($raw_link =~ m/\%[A-F0-9]{2}/i) {
-
-    if (($raw_link =~ m/\%[A-F0-9]{2}/i) || ($raw_link =~ m/\&\#x[0-9A-F]+;/i) || ($raw_link =~ m/\&\#[0-9]+;/i)) {
-	if (!-e $raw_link) {
-	    $raw_link = &unicode::url_decode($raw_link,1);
-	}
-    } 
-    
-    return $raw_link;
-}
-
-sub read_into_doc_obj 
-{
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-        
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    # Lookup content_encoding worked out in file_block pass for this file
-    # Store it under the local name 'content_encoding' so its nice and 
-    # easy to access
-    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
-
-    # get the input file
-    my $input_filename = $file;
-    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-    $suffix = lc($suffix);   
-    my $tidy_filename;
-    if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'}))
-    {
-	# because the document has to be sectionalized set the description tags 
-	$self->{'description_tags'} = 1;
-	
-	# set the file to be tidied
-	$input_filename = &util::filename_cat($base_dir,$file) if $base_dir =~ m/\w/;
-	
-	# get the tidied file
-	#my $tidy_filename = $self->tmp_tidy_file($input_filename);
-	$tidy_filename = $self->convert_tidy_or_oldHDL_file($input_filename);
-	
-	# derive tmp filename from input filename
-	my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($tidy_filename, "\\.[^\\.]+\$");
-	
-	# set the new input file and base_dir to be from the tidied file
-	$file = "$tailname$suffix";
-	$base_dir = $dirname;
-    }
-    
-    # call the parent read_into_doc_obj
-    my ($process_status,$doc_obj) = $self->SUPER::read_into_doc_obj($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli);
-    if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'}))
-    {
-	# now we need to reset the filenames in the doc obj so that the converted filenames are not used
-	my $collect_file = &util::filename_within_collection($filename_full_path);
-	$doc_obj->set_source_filename ($collect_file, $self->{'file_rename_method'}); 
-	## set_source_filename does not set the doc_obj source_path which is used in archives dbs for incremental
-	# build. So set it manually.
-	$doc_obj->set_source_path($filename_full_path);
-	my $collect_conv_file = &util::filename_within_collection($tidy_filename);
-	$doc_obj->set_converted_filename($collect_conv_file);
-
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-	my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-	$self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-    }
-
-    delete $self->{'store_content_encoding'}->{$filename_full_path};
-    $self->{'content_encoding'} = undef;
-
-    return ($process_status,$doc_obj);
-}
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    if ($ENV{'GSDLOS'} =~ m/^windows/i) {
-	# this makes life so much easier... perl can cope with unix-style '/'s.
-	$base_dir =~ s@(\\)+@/@g; 
-	$file =~ s@(\\)+@/@g;     
-    }
-
-    my $filename = &util::filename_cat($base_dir,$file);
-    my $upgraded_base_dir = &util::upgrade_if_dos_filename($base_dir);
-    my $upgraded_filename = &util::upgrade_if_dos_filename($filename);
-
-    if ($ENV{'GSDLOS'} =~ m/^windows/i) {
-	# And again
-	$upgraded_base_dir =~ s@(\\)+@/@g;	
-	$upgraded_filename =~ s@(\\)+@/@g;
-	
-	# Need to make sure there is a '/' on the end of upgraded_base_dir
-	if (($upgraded_base_dir ne "") && ($upgraded_base_dir !~ m/\/$/)) {
-	    $upgraded_base_dir .= "/";
-	}
-    }
-    my $upgraded_file = &util::filename_within_directory($upgraded_filename,$upgraded_base_dir);
-    
-    # reset per-doc stuff...
-    $self->{'aux_files'} = {};
-    $self->{'dir_num'} = 0;
-    $self->{'file_num'} = 0;
-
-    # process an HTML file where sections are divided by headings tags (H1, H2 ...)
-    # you can also include metadata in the format (X can be any number)
-    # <hX>Title<!--gsdl-metadata
-    #	<Metadata name="name1">value1</Metadata>
-    #	...
-    #	<Metadata name="nameN">valueN</Metadata>
-    #--></hX>
-    if ($self->{'sectionalise_using_h_tags'}) {
-	# description_tags should allways be activated because we convert headings to description tags
-	$self->{'description_tags'} = 1;
-
-	my $arrSections = [];
-	$$textref =~ s/<h([0-9]+)[^>]*>(.*?)<\/h[0-9]+>/$self->process_heading($1, $2, $arrSections, $upgraded_file)/isge;
-
-	if (scalar(@$arrSections)) {
-	    my $strMetadata = $self->update_section_data($arrSections, -1);
-	    if (length($strMetadata)) {
-		$strMetadata = '<!--' . $strMetadata . "\n-->\n</body>";
-		$$textref =~ s/<\/body>/$strMetadata/ig;
-	    }
-	}
-    }
-
-    my $cursection = $doc_obj->get_top_section();
-
-    $self->extract_metadata ($textref, $metadata, $doc_obj, $cursection)
-	unless $self->{'no_metadata'} || $self->{'description_tags'};
-
-    # Store URL for page as metadata - this can be used for an
-    # altavista style search interface. The URL won't be valid
-    # unless the file structure contains the domain name (i.e.
-    # like when w3mir is used to download a website).
-
-    # URL metadata (even invalid ones) are used to support internal
-    # links, so even if 'file_is_url' is off, still need to store info
-
-    my ($tailname,$dirname) = &File::Basename::fileparse($upgraded_file);
-
-#    my $utf8_file = $self->filename_to_utf8_metadata($file);
-#    $utf8_file =~ s/&\#095;/_/g;
-#    variable below used to be utf8_file
-
-    my $url_encoded_file = &unicode::raw_filename_to_url_encoded($tailname);
-    my $utf8_url_encoded_file = &unicode::raw_filename_to_utf8_url_encoded($tailname);
-
-    my $web_url = "http://";
-    my $utf8_web_url = "http://";
-    if(defined $dirname) { # local directory
-        # Check for "ftp" in the domain name of the directory
-        #  structure to determine if this URL should be a ftp:// URL
-        # This check is not infallible, but better than omitting the
-        #  check, which would cause all files downloaded from ftp sites
-        #  via mirroring with wget to have potentially erroneous http:// URLs
-        #  assigned in their metadata
-        if ($dirname =~ /^[^\/]*ftp/i)
-	{
-	  $web_url = "ftp://";
-	  $utf8_web_url = "ftp://";
-	}
-	$dirname = $self->eval_dir_dots($dirname);
-	$dirname .= &util::get_dirsep() if $dirname ne ""; # if there's a directory, it should end on "/"
-
-	$web_url = $web_url.$dirname.$url_encoded_file; 
-	$utf8_web_url = $utf8_web_url.$dirname.$utf8_url_encoded_file; 
-    } else {
-	$web_url = $web_url.$url_encoded_file;
-	$utf8_web_url = $utf8_web_url.$utf8_url_encoded_file;
-    }
-    $web_url =~ s/\\/\//g;
-    $utf8_web_url =~ s/\\/\//g;
-
-    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-	print STDERR "*******DEBUG: upgraded_file:       $upgraded_file\n";
-	print STDERR "*******DEBUG: adding URL metadata: $utf8_url_encoded_file\n";
-    }
-
-
-    $doc_obj->add_utf8_metadata($cursection, "URL", $web_url);
-    $doc_obj->add_utf8_metadata($cursection, "UTF8URL", $utf8_web_url);
-
-    if ($self->{'file_is_url'}) {
-	$doc_obj->add_metadata($cursection, "weblink", "<a href=\"$web_url\">");
-	$doc_obj->add_metadata($cursection, "webicon", "_iconworld_");
-	$doc_obj->add_metadata($cursection, "/weblink", "</a>");
-    }
-
-    if ($self->{'description_tags'}) {
-	# remove the html header - note that doing this here means any
-	# sections defined within the header will be lost (so all <Section>
-	# tags must appear within the body of the HTML)
-	my ($head_keep) = ($$textref =~ m/^(.*?)<body[^>]*>/is);
-
-	$$textref =~ s/^.*?<body[^>]*>//is;
-	$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
-
-	my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
-	my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
-
-	my $lt = '(?:<|&lt;)';
-	my $gt = '(?:>|&gt;)';
-	my $quot = '(?:"|&quot;|&rdquo;|&ldquo;)';
-
-	my $dont_strip = '';
-	if ($self->{'no_strip_metadata_html'}) {
-	    ($dont_strip = $self->{'no_strip_metadata_html'}) =~ s{,}{|}g;
-	}
-
-	my $found_something = 0; my $top = 1;
-	while ($$textref =~ s/^(.*?)$opencom(.*?)$closecom//s) {
-	    my $text = $1;
-	    my $comment = $2;
-	    if (defined $text) {
-		# text before a comment - note that getting to here
-		# doesn't necessarily mean there are Section tags in
-		# the document
-		$self->process_section(\$text, $upgraded_base_dir, $upgraded_file, $doc_obj, $cursection);
-	    }
-	    while ($comment =~ s/$lt(.*?)$gt//s) {
-		my $tag = $1;
-		if ($tag eq "Section") {
-		    $found_something = 1;
-		    $cursection = $doc_obj->insert_section($doc_obj->get_end_child($cursection)) unless $top;
-		    $top = 0;
-		} elsif ($tag eq "/Section") {
-		    $found_something = 1;
-		    $cursection = $doc_obj->get_parent_section ($cursection);
-		} elsif ($tag =~ m/^Metadata name=$quot(.*?)$quot/s) {
-		    my $metaname = $1;
-		    my $accumulate = $tag =~ m/mode=${quot}accumulate${quot}/ ? 1 : 0;
-		    $comment =~ s/^(.*?)$lt\/Metadata$gt//s;
-		    my $metavalue = $1;
-		    $metavalue =~ s/^\s+//;
-		    $metavalue =~ s/\s+$//;
-                    # assume that no metadata value intentionally includes
-                    # carriage returns or HTML tags (if they're there they
-                    # were probably introduced when converting to HTML from
-                    # some other format).
-		    # actually some people want to have html tags in their
-		    # metadata.
-		    $metavalue =~ s/[\cJ\cM]/ /sg;
-		    $metavalue =~ s/<[^>]+>//sg
-			unless $dont_strip && ($dont_strip eq 'all' || $metaname =~ m/^($dont_strip)$/);
-		    $metavalue =~ s/\s+/ /sg;
-		    if ($metaname =~ /\./) { # has a namespace
-			$metaname = "ex.$metaname";
-		    }
-		    if ($accumulate) {
-			$doc_obj->add_utf8_metadata($cursection, $metaname, $metavalue);
-		    } else {
-			$doc_obj->set_utf8_metadata_element($cursection, $metaname, $metavalue);	
-		    }
-		} elsif ($tag eq "Description" || $tag eq "/Description") {
-		    # do nothing with containing Description tags
-		} else {
-		    # simple HTML tag (probably created by the conversion
-		    # to HTML from some other format) - we'll ignore it and
-		    # hope for the best ;-)
-		}
-	    }
-	}
-	if ($cursection ne "") {
-	    print $outhandle "HTMLPlugin: WARNING: $upgraded_file contains unmatched <Section></Section> tags\n";
-	}
-
-	$$textref =~ s/^.*?<body[^>]*>//is;
-	$$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
-	if ($$textref =~ m/\S/) {
-	    if (!$found_something) {
-		if ($self->{'verbosity'} > 2) {
-		    print $outhandle "HTMLPlugin: WARNING: $upgraded_file appears to contain no Section tags so\n";
-		    print $outhandle "          will be processed as a single section document\n";
-		}
-
-		# go ahead and process single-section document
-		$self->process_section($textref, $upgraded_base_dir, $upgraded_file, $doc_obj, $cursection);
-
-		# if document contains no Section tags we'll go ahead
-		# and extract metadata (this won't have been done
-		# above as the -description_tags option prevents it)
-		my $complete_text = $head_keep.$doc_obj->get_text($cursection);
-		$self->extract_metadata (\$complete_text, $metadata, $doc_obj, $cursection) 
-		    unless $self->{'no_metadata'};
-
-	    } else {
-		print $outhandle "HTMLPlugin: WARNING: $upgraded_file contains the following text outside\n";
-		print $outhandle "          of the final closing </Section> tag. This text will\n";
-		print $outhandle "          be ignored.";
-
-		my ($text);
-		if (length($$textref) > 30) {
-		    $text = substr($$textref, 0, 30) . "...";
-		} else {
-		    $text = $$textref;
-		}
-		$text =~ s/\n/ /isg;
-		print $outhandle " ($text)\n";
-	    }
-	} elsif (!$found_something) {
-
-	    if ($self->{'verbosity'} > 2) {
-		# may get to here if document contained no valid Section
-		# tags but did contain some comments. The text will have
-		# been processed already but we should print the warning
-		# as above and extract metadata
-		print $outhandle "HTMLPlugin: WARNING: $upgraded_file appears to contain no Section tags and\n";
-		print $outhandle "          is blank or empty.  Metadata will be assigned if present.\n";
-	    }
-
-	    my $complete_text = $head_keep.$doc_obj->get_text($cursection);
-	    $self->extract_metadata (\$complete_text, $metadata, $doc_obj, $cursection) 
-		unless $self->{'no_metadata'};
-	}
-	
-    } else {
-
-	# remove header and footer
-	if (!$self->{'keep_head'} || $self->{'description_tags'}) {
-	    $$textref =~ s/^.*?<body[^>]*>//is;
-	    $$textref =~ s/(<\/body[^>]*>|<\/html[^>]*>)//isg;
-	}
-
-
-	# single section document
-	$self->process_section($textref, $upgraded_base_dir, $upgraded_file, $doc_obj, $cursection);
-    }
-
-    return 1;
-}
-
-
-sub process_heading
-{
-    my ($self, $nHeadNo, $strHeadingText, $arrSections, $file) = @_;
-    $strHeadingText = '' if (!defined($strHeadingText));
-
-    my $strMetadata = $self->update_section_data($arrSections, int($nHeadNo));
-
-    my $strSecMetadata = '';
-    while ($strHeadingText =~ s/<!--gsdl-metadata(.*?)-->//is)
-    {
-	$strSecMetadata .= $1;
-    }
-
-    $strHeadingText =~ s/^\s+//g;
-    $strHeadingText =~ s/\s+$//g;
-    $strSecMetadata =~ s/^\s+//g;
-    $strSecMetadata =~ s/\s+$//g;
-
-    $strMetadata .= "\n<Section>\n\t<Description>\n\t\t<Metadata name=\"Title\">" . $strHeadingText . "</Metadata>\n";
-
-    if (length($strSecMetadata)) {
-	$strMetadata .= "\t\t" . $strSecMetadata . "\n";
-    }
-
-    $strMetadata .= "\t</Description>\n";
-
-    return "<!--" . $strMetadata . "-->";
-}
-
-
-sub update_section_data
-{
-    my ($self, $arrSections, $nCurTocNo) = @_;
-    my ($strBuffer, $nLast, $nSections) = ('', 0, scalar(@$arrSections));
-
-    if ($nSections == 0) {
-	push @$arrSections, $nCurTocNo;
-	return $strBuffer;
-    }
-    $nLast = $arrSections->[$nSections - 1];
-    if ($nCurTocNo > $nLast) {
-	push @$arrSections, $nCurTocNo;
-	return $strBuffer;
-    }
-    for(my $i = $nSections - 1; $i >= 0; $i--) {
-	if ($nCurTocNo <= $arrSections->[$i]) {
-	    $strBuffer .= "\n</Section>";
-	    pop @$arrSections;
-	}
-    }
-    push @$arrSections, $nCurTocNo;
-    return $strBuffer;
-}
-
-
-# note that process_section may be called multiple times for a single
-# section (relying on the fact that add_utf8_text appends the text to any
-# that may exist already).
-sub process_section {
-    my $self = shift (@_);
-    my ($textref, $base_dir, $file, $doc_obj, $cursection) = @_;
-    # trap links
-    if (!$self->{'nolinks'}) {
-	# usemap="./#index" not handled correctly => change to "#index"
-##	$$textref =~ s/(<img[^>]*?usemap\s*=\s*[\"\']?)([^\"\'>\s]+)([\"\']?[^>]*>)/
-
-##	my $opencom = '(?:<!--|&lt;!(?:&mdash;|&#151;|--))';
-##	my $closecom = '(?:-->|(?:&mdash;|&#151;|--)&gt;)';
-
-	$$textref =~ s/(<img[^>]*?usemap\s*=\s*)((?:[\"][^\"]+[\"])|(?:[\'][^\']+[\'])|(?:[^\s\/>]+))([^>]*>)/
-	    $self->replace_usemap_links($1, $2, $3)/isge;
-
-	$$textref =~ s/(<(?:a|area|frame|link|script)\s+[^>]*?\s*(?:href|src)\s*=\s*)((?:[\"][^\"]+[\"])|(?:[\'][^\']+[\'])|(?:[^\s\/>]+))([^>]*>)/
-	    $self->replace_href_links ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge; 
-	
-##	$$textref =~ s/($opencom.*?)?+(<(?:a|area|frame|link|script)\s+[^>]*?\s*(?:href|src)\s*=\s*)((?:[\"][^\"]+[\"])|(?:[\'][^\']+[\'])|(?:[^\s\/>]+))([^>]*>)(.*?$closecom)?+/
-#	    $self->replace_href_links ($1, $2, $3, $4, $5, $base_dir, $file, $doc_obj, $cursection)/isge;
-    }
-
-    # trap images
-
-    # Previously, by default, HTMLPlugin would embed <img> tags inside anchor tags
-    # i.e. <a href="image><img src="image"></a> in order to overcome a problem that
-    # turned regular text succeeding images into links. That is, by embedding <imgs>
-    # inside <a href=""></a>, the text following images were no longer misbehaving.
-    # However, there would be many occasions whereby images were not meant to link
-    # to their source images but where the images would link to another web page.
-    # To allow this, the no_image_links option was introduced: it would prevent
-    # the behaviour of embedding images into links that referenced the source images.
-
-    # Somewhere along the line, the problem of normal text turning into links when 
-    # such text followed images which were not embedded in <a href=""></a> ceased 
-    # to occur. This is why the following lines have been commented out (as well as 
-    # two lines in replace_images). They appear to no longer apply.
-
-    # If at any time, there is a need for having images embedded in <a> anchor tags,
-    # then it might be better to turn that into an HTMLPlugin option rather than make
-    # it the default behaviour. Also, eventually, no_image_links needs to become
-    # a deprecated option for HTMLPlugin as it has now become the default behaviour.
-
-    #if(!$self->{'no_image_links'}){
-    $$textref =~ s/(<(?:img|embed|table|tr|td)[^>]*?(?:src|background)\s*=\s*)((?:[\"][^\"]+[\"])|(?:[\'][^\']+[\'])|(?:[^\s\/>]+))([^>]*>)/
-	$self->replace_images ($1, $2, $3, $base_dir, $file, $doc_obj, $cursection)/isge;
-    #}
-
-    # add text to document object
-    # turn \ into \\ so that the rest of greenstone doesn't think there
-    # is an escape code following. (Macro parsing loses them...)
-    $$textref =~ s/\\/\\\\/go;
-    
-    $doc_obj->add_utf8_text($cursection, $$textref);
-}
-
-sub replace_images {
-    my $self = shift (@_);
-    my ($front, $link, $back, $base_dir, 
-	$file, $doc_obj, $section) = @_;
-
-    # remove quotes from link at start and end if necessary
-    if ($link=~/^[\"\']/) {
-	$link=~s/^[\"\']//;
-	$link=~s/[\"\']$//;
-	$front.='"';
-	$back="\"$back";
-    }
-
-    $link =~ s/\n/ /g;
-
-    # Hack to overcome Windows wv 0.7.1 bug that causes embedded images to be broken
-    # If the Word file path has spaces in it, wv messes up and you end up with
-    #   absolute paths for the images, and without the "file://" prefix
-    # So check for this special case and massage the data to be correct
-    if ($ENV{'GSDLOS'} =~ m/^windows/i && $self->{'plugin_type'} eq "WordPlug" && $link =~ m/^[A-Za-z]\:\\/) {
-	$link =~ s/^.*\\([^\\]+)$/$1/;
-    }
-    
-    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
-
-    my $img_file =  $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section);
-
-#    print STDERR "**** link = $link\n**** href = $href\n**** img_file = $img_file, rl = $rl\n";
-
-    my $anchor_name = $img_file;
-    #$anchor_name =~ s/^.*\///;
-    #$anchor_name = "<a name=\"$anchor_name\" ></a>";
-
-    my $image_link = $front . $img_file .$back;
-    return $image_link;
-
-    # The reasons for why the following two lines are no longer necessary can be 
-    # found in subroutine process_section
-    #my $anchor_link = "<a href=\"$img_file\" >".$image_link."</a>";  
-    #return $anchor_link;	
-    
-    #return $front . $img_file . $back . $anchor_name;
-}
-
-sub replace_href_links {
-    my $self = shift (@_);
-    my ($front, $link, $back, $base_dir, $file, $doc_obj, $section) = @_;
-
-    # remove quotes from link at start and end if necessary
-    if ($link=~/^[\"\']/) {
-	$link=~s/^[\"\']//;
-	$link=~s/[\"\']$//;
-	$front.='"';
-	$back="\"$back";
-    }
-
-    # attempt to sort out targets - frames are not handled 
-    # well in this plugin and some cases will screw things
-    # up - e.g. the _parent target (so we'll just remove 
-    # them all ;-)
-    $front =~ s/(target=\"?)_top(\"?)/$1_gsdltop_$2/is;
-    $back =~ s/(target=\"?)_top(\"?)/$1_gsdltop_$2/is;
-    $front =~ s/target=\"?_parent\"?//is;
-    $back =~ s/target=\"?_parent\"?//is;
-
-    return $front . $link . $back if $link =~ m/^\#/s;
-    $link =~ s/\n/ /g;
-
-    # Find file referred to by $link on file system 
-    # This is more complicated than it sounds when char encodings
-    # is taken in to account
-    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
-
-    # href may use '\'s where '/'s should be on Windows
-    $href =~ s/\\/\//g;
-    my ($filename) = $href =~ m/^(?:.*?):(?:\/\/)?(.*)/;
-    
-
-    ##### leave all these links alone (they won't be picked up by intermediate 
-    ##### pages). I think that's safest when dealing with frames, targets etc.
-    ##### (at least until I think of a better way to do it). Problems occur with
-    ##### mailto links from within small frames, the intermediate page is displayed
-    ##### within that frame and can't be seen. There is still potential for this to
-    ##### happen even with html pages - the solution seems to be to somehow tell
-    ##### the browser from the server side to display the page being sent (i.e. 
-    ##### the intermediate page) in the top level window - I'm not sure if that's 
-    ##### possible - the following line should probably be deleted if that can be done
-    return $front . $link . $back if $href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/is;
-
-    if (($rl == 0) || ($filename =~ m/$self->{'process_exp'}/) || 
-	($href =~ m/\/$/) || ($href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
-
-	if ($ENV{'GSDLOS'} =~ m/^windows$/) {
-
-	    # Don't do any encoding for now, as not clear what
-	    # the right thing to do is to support filename
-	    # encoding on Windows when they are not UTF16
-	    # 
-	}
-	else {
-	    # => Unix-based system
-
-	    # If web page didn't give encoding, then default to utf8
-	    my $content_encoding= $self->{'content_encoding'} || "utf8";
-
-	    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-		print STDERR "**** Encoding with '$content_encoding', href: $href\n";
-	    }
-
-	    # on Darwin, the unicode filenames are stored on the file
-	    # system in decomposed form, so any href link (including when 
-	    # URL-encoded) should refer to the decomposed name of the file
-	    if ($ENV{'GSDLOS'} =~ /^darwin$/i) {
-		$href = normalize('D', $href); # Normalization Form D (decomposition) 
-	    }
-
-	    $href = encode($content_encoding,$href);
-	}
-
-	$href = &unicode::raw_filename_to_utf8_url_encoded($href);	
-	$href = &unicode::filename_to_url($href);
-
-	&ghtml::urlsafe ($href);
-
-	if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-		print STDERR "******DEBUG: href=$href\n";    
-	}
-
-
-	return $front . "_httpextlink_&amp;rl=" . $rl . "&amp;href=" . $href . $hash_part . $back;
-    } else {
-	# link is to some other type of file (e.g., an image) so we'll
-	# need to associate that file
-	return $front . $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) . $back;
-    }
-}
-
-sub add_file {
-    my $self = shift (@_);
-    my ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) = @_;
-    my ($newname);
-
-    my $filename = $href;
-    if ($base_dir eq "") {
-	if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
-	    # remove http://
-	    $filename =~ s/^[^:]*:\/\///;
-	}
-	else {
-	    # remove http:/ thereby leaving one slash at the start as
-	    # part of full pathname
-	    $filename =~ s/^[^:]*:\///;
-	}
-    }
-    else {
-	# remove http://
-	$filename =~ s/^[^:]*:\/\///;
-    }
-
-    $filename = &util::filename_cat($base_dir, $filename);
-
-    if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'})) {
-	# we are processing a tidytmp file - want paths to be in import
-	$filename =~ s/([\\\/])tidytmp([\\\/])/$1import$2/;
-    }
-
-    # Replace %XX's in URL with decoded value if required. Note that the
-    # filename may include the %XX in some situations. If the *original*
-    # file's name was in URL encoding, the following method will not decode
-    # it.
-    my $unicode_filename = $filename;
-    my $opt_decode_unicode_filename = $self->opt_url_decode($unicode_filename);
-
-    # wvWare can generate <img src="StrangeNoGraphicData"> tags, but with no
-    # (it seems) accompanying file
-    if ($opt_decode_unicode_filename =~ m/StrangeNoGraphicData$/) { return ""; }
-
-    my $content_encoding= $self->{'content_encoding'} || "utf8";
-
-    if ($ENV{'GSDLOS'} =~ /^(linux|solaris)$/i) {
-	# The filenames that come through the HTML file have been decoded
-	# into Unicode aware Perl strings.  Need to convert them back
-	# to their initial raw-byte encoding to match the file that
-	# exists on the file system
-	$filename = encode($content_encoding, $opt_decode_unicode_filename);
-    }
-    elsif ($ENV{'GSDLOS'} =~ /^darwin$/i) {
-	# HFS+ is UTF8 with decompostion
-	$filename = encode($content_encoding, $opt_decode_unicode_filename);
-	$filename = normalize('D', $filename); # Normalization Form D (decomposition)
-
-    }
-    elsif ($ENV{'GSDLOS'} =~ /^windows$/i) {
-	my $long_filename = Win32::GetLongPathName($opt_decode_unicode_filename);
-
-	if (defined $long_filename) {
-	    my $short_filename = Win32::GetLongPathName($long_filename);
-	    $filename = $short_filename;
-	}
-#	else {
-#	    print STDERR "***** failed to map href to real file:\n";
-#	    print STDERR "****** $href -> $opt_decode_unicode_filename\n";
-#	}
-    }
-    else {
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "Warning: Unrecognized operating system ", $ENV{'GSDLOS'}, "\n";
-	print $outhandle "         in file system encoding of href: $href\n";
-	print $outhandle "         No character encoding done.\n";
-    }
-
-
-    # some special processing if the intended filename was converted to utf8, but
-    # the actual file still needs to be renamed
-    if (!&util::fd_exists($filename)) {
-	# try the original filename stored in map
-	if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-		print STDERR "******!! orig filename did not exist: $filename\n";
-	}
-
-##	print STDERR "**** trying to look up unicode_filename: $unicode_filename\n";
-
-	my $original_filename = $self->{'unicode_to_original_filename'}->{$unicode_filename};
-
-	if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-		print STDERR "******   From lookup unicode_filename, now trying for: $original_filename\n";
-	}
-
-	if (defined $original_filename && -e $original_filename) {
-		if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-			print STDERR "******   Found match!\n";
-		}
-	    $filename = $original_filename;
-	}
-    }
-    
-    my ($ext) = $filename =~ m/(\.[^\.]*)$/;
-
-    if ($rl == 0) {
-	if ((!defined $ext) || ($ext !~ m/$self->{'assoc_files'}/)) {
-	    return "_httpextlink_&amp;rl=0&amp;el=prompt&amp;href=" . $href . $hash_part;
-	}
-	else {
-	    return "_httpextlink_&amp;rl=0&amp;el=direct&amp;href=" . $href . $hash_part;
-	}
-    }
-
-    if ((!defined $ext) || ($ext !~ m/$self->{'assoc_files'}/)) {
-	return "_httpextlink_&amp;rl=" . $rl . "&amp;href=" . $href . $hash_part;
-    }
-    # add the original image file as a source file
-    if (!$self->{'processing_tmp_files'} ) {
-	$doc_obj->associate_source_file($filename);
-    }
-    if ($self->{'rename_assoc_files'}) {
-	if (defined $self->{'aux_files'}->{$href}) {
-	    $newname = $self->{'aux_files'}->{$href}->{'dir_num'} . "/" .
-		$self->{'aux_files'}->{$href}->{'file_num'} . $ext;
-	} else {
-	    $newname = $self->{'dir_num'} . "/" . $self->{'file_num'} . $ext;
-	    $self->{'aux_files'}->{$href} = {'dir_num' => $self->{'dir_num'}, 'file_num' => $self->{'file_num'}};
-	    $self->inc_filecount ();
-	}
-	$doc_obj->associate_file($filename, $newname, undef, $section);
-	return "_httpdocimg_/$newname";
-    } else {
-	if(&unicode::is_url_encoded($unicode_filename)) {
-	    # use the possibly-decoded filename instead to avoid double URL encoding
-	    ($newname) = $filename =~ m/([^\/\\]*)$/;
-	} else {
-	    ($newname) = $unicode_filename =~ m/([^\/\\]*)$/;
-	}
-
-	# Make sure this name uses only ASCII characters. 
-	# We use either base64 or URL encoding, as these preserve original encoding
-	$newname = &util::rename_file($newname, $self->{'file_rename_method'});
-
-###	print STDERR "***** associating $filename (raw-byte/utf8)-> $newname\n";
-	$doc_obj->associate_file($filename, $newname, undef, $section);
-
-	# Since the generated image will be URL-encoded to avoid file-system/browser mess-ups
-	# of filenames, URL-encode the additional percent signs of the URL-encoded filename
-	my $newname_url = $newname;
-	$newname_url = &unicode::filename_to_url($newname_url);
-	return "_httpdocimg_/$newname_url";	
-    }
-}
-
-
-sub format_link {
-    my $self = shift (@_);
-    my ($link, $base_dir, $file) = @_;
-
-    # strip off hash part, e.g. #foo, but watch out for any entities, e.g. &#x3B1;
-    my ($before_hash, $hash_part) = $link =~ m/^(.*?[^&])(\#.*)?$/;
-	
-    $hash_part = "" if !defined $hash_part;
-    if (!defined $before_hash || $before_hash !~ m/[\w\.\/]/) {
-		my $outhandle = $self->{'outhandle'};
-		print $outhandle "HTMLPlugin: ERROR - badly formatted tag ignored ($link)\n"
-			if $self->{'verbosity'};
-		return ($link, "", 0);
-    }
-	
-    if ($before_hash =~ s@^((?:http|https|ftp|file|mms)://)@@i) {
-		my $type = $1;
-
-	if ($link =~ m/^(http|ftp):/i) {
-	    # Turn url (using /) into file name (possibly using \ on windows)
-	    my @http_dir_split = split('/', $before_hash);
-	    $before_hash = &util::filename_cat(@http_dir_split);
-	}
-
-	$before_hash = $self->eval_dir_dots($before_hash);
-	
-	my $linkfilename = &util::filename_cat ($base_dir, $before_hash);
-	
-	my $rl = 0;
-	$rl = 1 if (-e $linkfilename);
-
-	# make sure there's a slash on the end if it's a directory
-	if ($before_hash !~ m/\/$/) {
-	    $before_hash .= "/" if (-d $linkfilename);
-	}
-	return ($type . $before_hash, $hash_part, $rl);
-	
-    } elsif ($link !~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i && $link !~ m/^\//) {
-
-	if ($before_hash =~ s@^/@@ || $before_hash =~ m/\\/) {
-
-	    # the first directory will be the domain name if file_is_url
-	    # to generate archives, otherwise we'll assume all files are
-	    # from the same site and base_dir is the root
-
-	    if ($self->{'file_is_url'}) {
-		my @dirs = split /[\/\\]/, $file;
-		my $domname = shift (@dirs);
-		$before_hash = &util::filename_cat($domname, $before_hash);
-		$before_hash =~ s@\\@/@g; # for windows
-	    }
-	    else
-	    {
-		# see if link shares directory with source document
-		# => turn into relative link if this is so!
-		
-		if ($ENV{'GSDLOS'} =~ m/^windows/i) {
-		    # too difficult doing a pattern match with embedded '\'s...
-		    my $win_before_hash=$before_hash;
-		    $win_before_hash =~ s@(\\)+@/@g;
-		    # $base_dir is already similarly "converted" on windows.
-		    if ($win_before_hash =~ s@^$base_dir/@@o) {
-			# if this is true, we removed a prefix
-			$before_hash=$win_before_hash;
-		    }
-		}
-		else {
-		    # before_hash has lost leading slash by this point,
-		    # -> add back in prior to substitution with $base_dir
-		    $before_hash = "/$before_hash"; 
-
-		    $before_hash = &util::filename_cat("",$before_hash);
-		    $before_hash =~ s@^$base_dir/@@;
-		}
-	    }
-	} else {
-	    # Turn relative file path into full path
-	    my $dirname = &File::Basename::dirname($file);
-	    $before_hash = &util::filename_cat($dirname, $before_hash);
-	    $before_hash = $self->eval_dir_dots($before_hash);	    	    
-	}
-
-	my $linkfilename = &util::filename_cat ($base_dir, $before_hash); 
-
-
-#	print STDERR "**** linkfilename = $linkfilename\n";
-#	if (!&util::fd_exists($linkfilename)) {
-#	    print STDERR "***** Warning: Could not find $linkfilename\n";
-#	}
-
-
-	# make sure there's a slash on the end if it's a directory
-	if ($before_hash !~ m/\/$/) {
-	    $before_hash .= "/" if (-d $linkfilename);
-	}
-
-#	print STDERR "*** returning: $before_hash\n";
-
-	return ("http://" . $before_hash, $hash_part, 1);
-    } else {
-	# mailto, news, nntp, telnet, javascript or gopher link
-	return ($before_hash, "", 0);
-    }
-}
-
-sub extract_first_NNNN_characters {
-    my $self = shift (@_);
-    my ($textref, $doc_obj, $thissection) = @_;
-    
-    foreach my $size (split /,/, $self->{'first'}) {
-	my $tmptext =  $$textref;
-	# skip to the body
-	$tmptext =~ s/.*<body[^>]*>//i;
-	# remove javascript
-	$tmptext =~ s@<script.*?</script>@ @sig;
-	$tmptext =~ s/<[^>]*>/ /g;
-	$tmptext =~ s/&nbsp;/ /g;
-	$tmptext =~ s/^\s+//;
-	$tmptext =~ s/\s+$//;
-	$tmptext =~ s/\s+/ /gs;
-	$tmptext = &unicode::substr ($tmptext, 0, $size);
-	$tmptext =~ s/\s\S*$/&#8230;/; # adds an ellipse (...)
-	$doc_obj->add_utf8_metadata ($thissection, "First$size", $tmptext);
-    }
-}
-
-
-sub extract_metadata {
-    my $self = shift (@_);
-    my ($textref, $metadata, $doc_obj, $section) = @_;
-    my $outhandle = $self->{'outhandle'};
-    # if we don't want metadata, we may as well not be here ...
-    return if (!defined $self->{'metadata_fields'});
-    
-    my $separator = $self->{'metadata_field_separator'};
-    if ($separator eq "") {
-	undef $separator;
-    }
-
-    # metadata fields to extract/save. 'key' is the (lowercase) name of the
-    # html meta, 'value' is the metadata name for greenstone to use
-    my %find_fields = ();
-
-    my %creator_fields = (); # short-cut for lookups
-
-
-    foreach my $field (split /,/, $self->{'metadata_fields'}) {
-        $field =~ s/^\s+//; # remove leading whitespace
-        $field =~ s/\s+$//; # remove trailing whitespace
-	
-	# support tag<tagname>
-	if ($field =~ m/^(.*?)\s*<(.*?)>$/) {
-	    # "$2" is the user's preferred gs metadata name
-	    $find_fields{lc($1)}=$2; # lc = lowercase
-	} else { # no <tagname> for mapping
-	    # "$field" is the user's preferred gs metadata name
-	    $find_fields{lc($field)}=$field; # lc = lowercase
-	}
-    }
-
-    if (defined $self->{'hunt_creator_metadata'} &&
-	$self->{'hunt_creator_metadata'} == 1 ) {
-	my @extra_fields =
-	    (
-	     'author',
-	     'author.email',
-	     'creator',
-	     'dc.creator',
-	     'dc.creator.corporatename',
-	     );
-
-	# add the creator_metadata fields to search for
-	foreach my $field (@extra_fields) {
-	    $creator_fields{$field}=0; # add to lookup hash
-	}
-    }
-
-
-    # find the header in the html file, which has the meta tags
-    $$textref =~ m@<head>(.*?)</head>@si;
-
-    my $html_header=$1;
-
-    # go through every <meta... tag defined in the html and see if it is
-    # one of the tags we want to match.
-    
-    # special case for title - we want to remember if its been found
-    my $found_title = 0;
-    # this assumes that ">" won't appear. (I don't think it's allowed to...)
-    $html_header =~ m/^/; # match the start of the string, for \G assertion
-    
-    while ($html_header =~ m/\G.*?<meta(.*?)>/sig) {
-	my $metatag=$1;
-	my ($tag, $value);
-
-	# find the tag name
-	$metatag =~ m/(?:name|http-equiv)\s*=\s*([\"\'])?(.*?)\1/is;
-	$tag=$2;
-	# in case they're not using " or ', but they should...
-	if (! $tag) {
-	    $metatag =~ m/(?:name|http-equiv)\s*=\s*([^\s\>]+)/is;
-	    $tag=$1;
-	}
-
-	if (!defined $tag) {
-	    print $outhandle "HTMLPlugin: can't find NAME in \"$metatag\"\n";
-	    next;
-	}
-	
-	# don't need to assign this field if it was passed in from a previous 
-	# (recursive) plugin
-	if (defined $metadata->{$tag}) {next}
-
-	# find the tag content
-	$metatag =~ m/content\s*=\s*([\"\'])?(.*?)\1/is;
-	$value=$2;
-
-	# The following code assigns the metaname to value if value is
-	# empty. Why would we do this?
-	#if (! $value) {
-	#    $metatag =~ m/(?:name|http-equiv)\s*=\s*([^\s\>]+)/is;
-	#    $value=$1;
-	#}
-	if (!defined $value || $value eq "") {
-	    print $outhandle "HTMLPlugin: can't find VALUE in <meta $metatag >\n" if ($self->{'verbosity'} > 2);
-	    next;
-	}
-	
-	# clean up and add
-	$value =~ s/\s+/ /gs;
-	chomp($value); # remove trailing \n, if any
-	if (exists $creator_fields{lc($tag)}) {
-	    # map this value onto greenstone's "Creator" metadata
-	    $tag='Creator';
-	} elsif (!exists $find_fields{lc($tag)}) {
-	    next; # don't want this tag
-	} else {
-	    # get the user's preferred capitalisation
-	    $tag = $find_fields{lc($tag)};
-	}
-	if (lc($tag) eq "title") {
-	    $found_title = 1;
-	}
-
-	if ($self->{'verbosity'} > 2) {
-	    print $outhandle " extracted \"$tag\" metadata \"$value\"\n";
-	}
-
-	if ($tag =~ /\./) {
-	    # there is a . so has a namespace, add ex.
-	    $tag = "ex.$tag";
-	}
-	if (defined $separator) {
-	    my @values = split($separator, $value);
-	    foreach my $v (@values) {
-		$doc_obj->add_utf8_metadata($section, $tag, $v) if $v =~ /\S/;
-	    }
-	}
-	else {
-	    $doc_obj->add_utf8_metadata($section, $tag, $value);
-	}
-    }
-    
-    # TITLE: extract the document title
-    if (exists $find_fields{'title'} && !$found_title) {
-	# we want a title, and didn't find one in the meta tags
-	# see if there's a <title> tag
-	my $title;
-	my $from = ""; # for debugging output only
-	if ($html_header =~ m/<title[^>]*>([^<]+)<\/title[^>]*>/is) {
-	    $title = $1;
-	    $from = "<title> tags";
-	}
-
-	if (!defined $title) {
-	    $from = "first 100 chars";
-	    # if no title use first 100 or so characters
-	    $title = $$textref;
-	    $title =~ s/^\xFE\xFF//; # Remove unicode byte order mark	    
-	    $title =~ s/^.*?<body>//si;
-	    # ignore javascript!
-	    $title =~ s@<script.*?</script>@ @sig;
-	    $title =~ s/<\/([^>]+)><\1>//g; # (eg) </b><b> - no space
-	    $title =~ s/<[^>]*>/ /g; # remove all HTML tags
-	    $title = substr ($title, 0, 100);
-	    $title =~ s/\s\S*$/.../;
-	}
-	$title =~ s/<[^>]*>/ /g; # remove html tags
-	$title =~ s/&nbsp;/ /g;
-	$title =~ s/(?:&nbsp;|\xc2\xa0)/ /g; # utf-8 for nbsp...
-	$title =~ s/\s+/ /gs; # collapse multiple spaces
-	$title =~ s/^\s*//;   # remove leading spaces
-	$title =~ s/\s*$//;   # remove trailing spaces
-
-	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
-	$title =~ s/^\s+//s; # in case title_sub introduced any...
-	$doc_obj->add_utf8_metadata ($section, "Title", $title);
-	print $outhandle " extracted Title metadata \"$title\" from $from\n" 
-	    if ($self->{'verbosity'} > 2);
-    } 
-    
-    # add FileFormat metadata
-    $doc_obj->add_metadata($section,"FileFormat", "HTML");
-
-    # Special, for metadata names such as tagH1 - extracts
-    # the text between the first <H1> and </H1> tags into "H1" metadata.
-
-    foreach my $field (keys %find_fields) {
-	if ($field !~ m/^tag([a-z0-9]+)$/i) {next}
-	my $tag = $1;
-	if ($$textref =~ m@<$tag[^>]*>(.*?)</$tag[^>]*>@g) {
-	    my $content = $1;
-	    $content =~ s/&nbsp;/ /g;
-	    $content =~ s/<[^>]*>/ /g;
-	    $content =~ s/^\s+//;
-	    $content =~ s/\s+$//;
-	    $content =~ s/\s+/ /gs;
-	    if ($content) {
-		$tag=$find_fields{"tag$tag"}; # get the user's capitalisation
-		$tag =~ s/^tag//i;
-		$doc_obj->add_utf8_metadata ($section, $tag, $content);
-		print $outhandle " extracted \"$tag\" metadata \"$content\"\n" 
-		    if ($self->{'verbosity'} > 2);
-	    }
-	}
-    }    
-}
-
-
-# evaluate any "../" to next directory up
-# evaluate any "./" as here
-sub eval_dir_dots {
-    my $self = shift (@_);
-    my ($filename) = @_;
-    my $dirsep_os = &util::get_os_dirsep();
-    my @dirsep = split(/$dirsep_os/,$filename);
-
-    my @eval_dirs = ();
-    foreach my $d (@dirsep) {
-	if ($d eq "..") {
-	    pop(@eval_dirs);
-	    
-	} elsif ($d eq ".") {
-	    # do nothing!
-
-	} else {
-	    push(@eval_dirs,$d);
-	}
-    }
-
-    # Need to fiddle with number of elements in @eval_dirs if the
-    # first one is the empty string.  This is because of a
-    # modification to util::filename_cat that supresses the addition
-    # of a leading '/' character (or \ if windows) (intended to help
-    # filename cat with relative paths) if the first entry in the
-    # array is the empty string.  Making the array start with *two*
-    # empty strings is a way to defeat this "smart" option.
-    #
-    if (scalar(@eval_dirs) > 0) {
-	if ($eval_dirs[0] eq ""){
-	    unshift(@eval_dirs,"");
-	}
-    }
-    
-    my $evaluated_filename = (scalar @eval_dirs > 0) ? &util::filename_cat(@eval_dirs) : "";
-    return $evaluated_filename;
-}
-
-sub replace_usemap_links {
-    my $self = shift (@_);
-    my ($front, $link, $back) = @_;
-
-    # remove quotes from link at start and end if necessary
-    if ($link=~/^[\"\']/) {
-	$link=~s/^[\"\']//;
-	$link=~s/[\"\']$//;
-	$front.='"';
-	$back="\"$back";
-    }
-
-    $link =~ s/^\.\///;
-    return $front . $link . $back;
-}
-
-sub inc_filecount {
-    my $self = shift (@_);
-
-    if ($self->{'file_num'} == 1000) {
-	$self->{'dir_num'} ++;
-	$self->{'file_num'} = 0;
-    } else {
-	$self->{'file_num'} ++;
-    }
-}
-
-
-# Extend read_file so that strings like &eacute; are
-# converted to UTF8 internally.  
-#
-# We don't convert &lt; or &gt; or &amp; or &quot; in case
-# they interfere with the GML files
-
-sub read_file {
-    my $self = shift(@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-
-    $self->SUPER::read_file($filename, $encoding, $language, $textref);
-
-    # Convert entities to their Unicode code-point equivalents
-    $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
-    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,1)/gseo;
-    $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;
-
-}
-
-sub HB_read_html_file {
-    my $self = shift (@_);
-    my ($htmlfile, $text) = @_;
-    
-    # load in the file
-    if (!open (FILE, $htmlfile)) {
-	print STDERR "ERROR - could not open $htmlfile\n";
-	return;
-    }
-
-    my $foundbody = 0;
-    $self->HB_gettext (\$foundbody, $text, "FILE");
-    close FILE;
-    
-    # just in case there was no <body> tag
-    if (!$foundbody) {
-	$foundbody = 1;
-	open (FILE, $htmlfile) || return;
-	$self->HB_gettext (\$foundbody, $text, "FILE");	
-	close FILE;
-    }
-    # text is in utf8
-}		
-
-# converts the text to utf8, as ghtml does that for &eacute; etc.
-sub HB_gettext {
-    my $self = shift (@_);
-    my ($foundbody, $text, $handle) = @_;
-    
-    my $line = "";
-    while (defined ($line = <$handle>)) {
-	# look for body tag
-	if (!$$foundbody) {
-	    if ($line =~ s/^.*<body[^>]*>//i) {
-		$$foundbody = 1;
-	    } else {
-		next;
-	    }
-	}
-	
-	# check for symbol fonts
-	if ($line =~ m/<font [^>]*?face\s*=\s*\"?(\w+)\"?/i) {
-	    my $font = $1;
-	    print STDERR "HBPlug::HB_gettext - warning removed font $font\n" 
-		if ($font !~ m/^arial$/i);
-	}
-
-	$$text .= $line;
-    }
-
-    if ($self->{'input_encoding'} eq "iso_8859_1") {
-	# convert to utf-8
-	$$text=&unicode::unicode2utf8(&unicode::convert2unicode("iso_8859_1", $text));
-    }
-    # convert any alphanumeric character entities to their utf-8
-    # equivalent for indexing purposes
-    #&ghtml::convertcharentities ($$text);
-
-    $$text =~ s/\s+/ /g; # remove \n's
-
-    # At this point $$text is a binary byte string
-    # => turn it into a Unicode aware string, so full
-    # Unicode aware pattern matching can be used.
-    # For instance: 's/\x{0101}//g' or '[[:upper:]]'
-    # 
-
-    $$text = decode("utf8",$$text);
-}
-
-sub HB_clean_section {
-    my $self = shift (@_);
-    my ($section) = @_;
-
-    # remove tags without a starting tag from the section
-    my ($tag, $tagstart);
-    while ($section =~ m/<\/([^>]{1,10})>/) {
-	$tag = $1;
-	$tagstart = index($section, "<$tag");
-	last if (($tagstart >= 0) && ($tagstart < index($section, "<\/$tag")));
-	$section =~ s/<\/$tag>//;
-    }
-    
-    # remove extra paragraph tags
-    while ($section =~ s/<p\b[^>]*>\s*<p\b/<p/ig) {}
-    
-    # remove extra stuff at the end of the section
-    while ($section =~ s/(<u>|<i>|<b>|<p\b[^>]*>|&nbsp;|\s)$//i) {}
-    
-    # add a newline at the beginning of each paragraph
-    $section =~ s/(.)\s*<p\b/$1\n\n<p/gi;
-    
-    # add a newline every 80 characters at a word boundary
-    # Note: this regular expression puts a line feed before
-    # the last word in each section, even when it is not
-    # needed.
-    $section =~ s/(.{1,80})\s/$1\n/g;
-    
-    # fix up the image links
-    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/
-	<center><img src=\"$1\" \/><\/center><br\/>/ig;
-    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/
-	<center><img src=\"$1\" \/><\/center><br\/>/ig;
-
-    return $section;
-}
-
-# Will convert the oldHDL format to the new HDL format (using the Section tag)	
-sub convert_to_newHDLformat
-{
-    my $self = shift (@_);
-    my ($file,$cnfile) = @_;
-    my $input_filename = $file;
-    my $tmp_filename = $cnfile;
-    
-    # write HTML tmp file with new HDL format
-    open (PROD, ">$tmp_filename") || die("Error Writing to File: $tmp_filename $!");
-    
-    # read in the file and do basic html cleaning (removing header etc)
-    my $html = "";
-    $self->HB_read_html_file ($input_filename, \$html);
-    
-    # process the file one section at a time
-    my $curtoclevel = 1;
-    my $firstsection = 1;
-    my $toclevel = 0;
-    while (length ($html) > 0) {
-	if ($html =~ s/^.*?(?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC(\d+)&gt;&gt;\s*(.*?)<p\b/<p/i) {
-	    $toclevel = $3;
-	    my $title = $4;
-	    my $sectiontext = "";
-	    if ($html =~ s/^(.*?)((?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC\d+&gt;&gt;)/$2/i) {
-		$sectiontext = $1;
-	    } else {
-		$sectiontext = $html;
-		$html = "";
-	    }
-
-	    # remove tags and extra spaces from the title
-	    $title =~ s/<\/?[^>]+>//g;
-	    $title =~ s/^\s+|\s+$//g;
-
-	    # close any sections below the current level and
-	    # create a new section (special case for the firstsection)
-	    print PROD "<!--\n";
-	    while (($curtoclevel > $toclevel) ||
-		   (!$firstsection && $curtoclevel == $toclevel)) {
-		$curtoclevel--;
-		print PROD "</Section>\n";
-	    }
-	    if ($curtoclevel+1 < $toclevel) {
-		print STDERR "WARNING - jump in toc levels in $input_filename " . 
-		    "from $curtoclevel to $toclevel\n";
-	    }
-	    while ($curtoclevel < $toclevel) {
-		$curtoclevel++;
-	    }
-
-	    if ($curtoclevel == 1) {
-	    	# add the header tag
-		print PROD "-->\n";
-    		print PROD "<HTML>\n<HEAD>\n<TITLE>$title</TITLE>\n</HEAD>\n<BODY>\n";
-		print PROD "<!--\n";
-	    }
-	    
-	    print PROD "<Section>\n\t<Description>\n\t\t<Metadata name=\"Title\">$title</Metadata>\n\t</Description>\n";
-	    
-	    print PROD "-->\n";
-	    
-	    # clean up the section html
-	    $sectiontext = $self->HB_clean_section($sectiontext);
-
-	    print PROD "$sectiontext\n";	 
-
-	} else {
-	    print STDERR "WARNING - leftover text\n" , $self->shorten($html), 
-	    "\nin $input_filename\n";
-	    last;
-	}
-	$firstsection = 0;
-    }
-    
-    print PROD "<!--\n";
-    while ($curtoclevel > 0) {
-	$curtoclevel--;
-	print PROD "</Section>\n";
-    }
-    print PROD "-->\n";
-    
-    close (PROD) || die("Error Closing File: $tmp_filename $!");
-    
-    return $tmp_filename;
-}		
-
-sub shorten {
-    my $self = shift (@_);
-    my ($text) = @_;
-
-    return "\"$text\"" if (length($text) < 100);
-
-    return "\"" . substr ($text, 0, 50) . "\" ... \"" . 
-	substr ($text, length($text)-50) . "\"";
-}
-
-sub convert_tidy_or_oldHDL_file
-{
-    my $self = shift (@_);
-    my ($file) = @_;
-    my $input_filename = $file;
-    
-    if (-d $input_filename)
-    {
-    	return $input_filename;
-    }
-    
-    # get the input filename
-    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-    my $base_dirname = $dirname;
-    $suffix = lc($suffix);   
-    
-    # derive tmp filename from input filename
-    # Remove any white space from filename -- no risk of name collision, and
-    # makes later conversion by utils simpler. Leave spaces in path...
-    # tidy up the filename with space, dot, hyphen between
-    $tailname =~ s/\s+//g; 
-    $tailname =~ s/\.+//g;
-    $tailname =~ s/\-+//g;
-    # convert to utf-8 otherwise we have problems with the doc.xml file
-    # later on
-    &unicode::ensure_utf8(\$tailname);
-    
-    # softlink to collection tmp dir
-    my $tmp_dirname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tidytmp");
-    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
-    
-    my $test_dirname = "";
-    my $f_separator = &util::get_os_dirsep();
-    
-    if ($dirname =~ m/import$f_separator/)
-    {
-    	$test_dirname = $'; #'
-	
-	#print STDERR "init $'\n";
-	
-	while ($test_dirname =~ m/[$f_separator]/)
-	{
-	    my $folderdirname = $`;
-	    $tmp_dirname = &util::filename_cat($tmp_dirname,$folderdirname);
-	    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
-	    $test_dirname = $'; #'
-	}
-    }
-    
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
-    
-    # tidy or convert the input file if it is a HTML-like file or it is accepted by the process_exp
-    if (($suffix eq ".htm") || ($suffix eq ".html") || ($suffix eq ".shtml"))
-    {   
-    	#convert the input file to a new style HDL
-    	my $hdl_output_filename = $input_filename;
-    	if ($self->{'old_style_HDL'})
-    	{
-	    $hdl_output_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
-	    $hdl_output_filename = $self->convert_to_newHDLformat($input_filename,$hdl_output_filename);
-    	}
-	
-	#just for checking copy all other file from the base dir to tmp dir if it is not exists
-	opendir(DIR,$base_dirname) or die "Can't open base directory : $base_dirname!";
-	my @files = grep {!/^\.+$/} readdir(DIR);
-	close(DIR);
-
-	foreach my $file (@files) 
-	{
-	    my $src_file = &util::filename_cat($base_dirname,$file);
-	    my $dest_file = &util::filename_cat($tmp_dirname,$file);
-	    if ((!-e $dest_file) && (!-d $src_file))
-	    {
-		# just copy the original file back to the tmp directory
-		copy($src_file,$dest_file) or die "Can't copy file $src_file to $dest_file $!";
-	    }
-	}
-	
-	# tidy the input file
-	my $tidy_output_filename = $hdl_output_filename;
-	if ($self->{'use_realistic_book'})
-	{
-	    $tidy_output_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
-	    $tidy_output_filename = $self->tmp_tidy_file($hdl_output_filename,$tidy_output_filename);
-	}
-	$tmp_filename = $tidy_output_filename;
-    }
-    else
-    {
-    	if (!-e $tmp_filename)
-	{
-	    # just copy the original file back to the tmp directory
-	    copy($input_filename,$tmp_filename) or die "Can't copy file $input_filename to $tmp_filename $!";
-	}
-    }
-    
-    return $tmp_filename;
-}
-
-
-# Will make the html input file as a proper XML file with removed font tag and
-# image size added to the img tag.
-# The tidying process takes place in a collection specific 'tmp' directory so 
-# that we don't accidentally damage the input.
-sub tmp_tidy_file 
-{
-    my $self = shift (@_);
-    my ($file,$cnfile) = @_;
-    my $input_filename = $file;
-    my $tmp_filename = $cnfile;
-    
-    # get the input filename
-    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-
-    require HTML::TokeParser::Simple;
-    
-    # create HTML parser to decode the input file
-    my $parser = HTML::TokeParser::Simple->new($input_filename);
-
-    # write HTML tmp file without the font tag and image size are added to the img tag
-    open (PROD, ">$tmp_filename") || die("Error Writing to File: $tmp_filename $!");
-    while (my $token = $parser->get_token())
-    {
-	# is it an img tag
-	if ($token->is_start_tag('img'))
-	{
-	    # get the attributes
-	    my $attr = $token->return_attr;
-
-	    # get the full path to the image
-	    my $img_file = &util::filename_cat($dirname,$attr->{src});
-	    
-	    # set the width and height attribute
-	    ($attr->{width}, $attr->{height}) = imgsize($img_file);
-
-	    # recreate the tag
-	    print PROD "<img";
-	    print PROD map { qq { $_="$attr->{$_}"} } keys %$attr;
-	    print PROD ">";
-	}
-	# is it a font tag
-	else 
-	{
-	    if (($token->is_start_tag('font')) || ($token->is_end_tag('font')))
-	    {
-		# remove font tag
-		print PROD "";
-	    }
-	    else
-	    {
-		# print without changes
-		print PROD $token->as_is;
-	    }
-	}
-    }
-    close (PROD) || die("Error Closing File: $tmp_filename $!");
-
-    # run html-tidy on the tmp file to make it a proper XML file
-
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "Converting HTML to be XML compliant:\n";
-
-    my $tidy_cmd = "tidy";
-    $tidy_cmd .= " -q" if ($self->{'verbosity'} <= 2);
-    $tidy_cmd .= " -raw -wrap 0 -asxml \"$tmp_filename\"";
-    if ($self->{'verbosity'} <= 2) {
-	if ($ENV{'GSDLOS'} =~ m/^windows/i) {
-	    $tidy_cmd .= " 2>nul";
-	}
-	else {
-	    $tidy_cmd .= " 2>/dev/null";
-	}
-	print $outhandle "  => $tidy_cmd\n";
-    }
-
-    my $tidyfile = `$tidy_cmd`;
-
-    # write result back to the tmp file
-    open (PROD, ">$tmp_filename") || die("Error Writing to File: $tmp_filename $!");
-    print PROD $tidyfile;
-    close (PROD) || die("Error Closing File: $tmp_filename $!");
-    
-    # return the output filename
-    return $tmp_filename;
-} 
-
-sub associate_cover_image
-{
-    my $self = shift(@_);
-    my ($doc_obj, $filename) = @_;
-    if (($self->{'use_realistic_book'}) || ($self->{'old_style_HDL'}))
-    {
-	# we will have cover image in tidytmp, but want it from import
-	$filename =~ s/([\\\/])tidytmp([\\\/])/$1import$2/;
-    }
-    $self->SUPER::associate_cover_image($doc_obj, $filename);
-}
-
-    
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ISISPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ISISPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,447 +1,0 @@
-###########################################################################
-#
-# ISISPlugin.pm -- A plugin for CDS/ISIS databases
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 1999-2004 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ISISPlugin;
-
-
-use multiread;
-use SplitTextFile;
-use MetadataRead;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-# ISISPlugin is a sub-class of SplitTextFile.
-# methods with identical signatures take precedence in the order given in the ISA list.
-sub BEGIN {
-    @ISISPlugin::ISA = ('MetadataRead', 'SplitTextFile');
-}
-
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_block_exp(),
-	'hiddengli' => "yes" },
-      { 'name' => "split_exp",
-	'desc' => "{SplitTextFile.split_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_split_exp(),
-        'hiddengli' => "yes" },
-
-      # The interesting options
-      { 'name' => "entry_separator",
-	'desc' => "{ISISPlugin.entry_separator}",
-	'type' => "string",
-	'reqd' => "no",
-	'deft' => "<br>" },
-      { 'name' => "subfield_separator",
-	'desc' => "{ISISPlugin.subfield_separator}",
-	'type' => "string",
-	'reqd' => "no",
-	'deft' => ", " }
-      ];
-
-my $options = { 'name'     => "ISISPlugin",
-		'desc'     => "{ISISPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-
-# This plugin processes files with the suffix ".mst"
-sub get_default_process_exp {
-    return q^(?i)(\.mst)$^;
-}
-
-
-# This plugin blocks files with the suffix ".fdt" and ".xrf"
-sub get_default_block_exp {
-    return q^(?i)(\.fdt|\.xrf)$^;
-    #return "";
-}
-
-    
-# This plugin splits the input text at the "----------" lines
-sub get_default_split_exp {
-    return q^\r?\n----------\r?\n^;
-}
-
-
-sub new
-{
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    # isis plug doesn't care about encoding - it assumes ascii unless the user
-    # has specified an encoding
-    if ($self->{'input_encoding'} eq "auto") {
-	$self->{'input_encoding'} = "ascii";
-    }
-    return bless $self, $class;
-}
-
-# we block the corresponding fdt and xrf
-# a pain on windows. blocks xxx.FDT, but if actual file is xx.fdt then 
-# complains that no plugin can process it. Have put it back to using 
-# block exp for now
-# This works now, as are doing case insenstive blocking on windows. However,
-# a pain for GLI as will not know what plugin processes the fdt and xrf.
-# if add to process expression, then get more problems.
-sub store_block_files_tmp {
-    
-    my $self =shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-    print STDERR "in store block files\n";
-    $self->check_auxiliary_files($filename_full_path);
-    if (-e $self->{'fdt_file_path'}) {
-	print STDERR "$self->{'fdt_file_path'}\n";
-	my $fdt_file = $self->{'fdt_file_path'};
-	&util::block_filename($block_hash,$fdt_file);
-    }
-    if (-e $self->{'xrf_file_path'}) {
-	print STDERR "$self->{'xrf_file_path'}\n";
-	my $xrf_file = $self->{'xrf_file_path'};
-	&util::block_filename($block_hash,$xrf_file);
-    }
-    
-
-}
-
-sub check_auxiliary_files {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my ($database_file_path_root) = ($filename =~ /(.*)\.mst$/i);
-    # Check the associated .fdt and .xrf files exist
-    $self->{'fdt_file_path'} = $database_file_path_root . ".FDT";
-    if (!-e $self->{'fdt_file_path'}) {
-	$self->{'fdt_file_path'} = $database_file_path_root . ".fdt";
-    }
-    $self->{'xrf_file_path'} = $database_file_path_root . ".XRF";
-    if (!-e $self->{'xrf_file_path'}) {
-	$self->{'xrf_file_path'} = $database_file_path_root . ".xrf";
-    }
-}
-    
-
-sub read_file
-{
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my ($database_file_path_root) = ($filename =~ /(.*)\.mst$/i);
-    my $mst_file_path_relative = $filename;
-    $mst_file_path_relative =~ s/^.+import.(.*?)$/$1/;
-
-    # Check the associated .fdt and .xrf files exist
-    $self->check_auxiliary_files($filename);
-    
-    if (!-e $self->{'fdt_file_path'}) {
-	print STDERR "<ProcessingError n='$mst_file_path_relative' r='Could not find ISIS FDT file $self->{'fdt_file_path'}'>\n" if ($self->{'gli'});
-	print $outhandle "Error: Could not find ISIS FDT file " . $self->{'fdt_file_path'} . ".\n";
-	return;
-    }
-    if (!-e $self->{'xrf_file_path'}) {
-	print STDERR "<ProcessingError n='$mst_file_path_relative' r='Could not find ISIS XRF file $self->{'xrf_file_path'}'>\n" if ($self->{'gli'});
-	print $outhandle "Error: Could not find ISIS XRF file " . $self->{'xrf_file_path'} . ".\n";
-	return;
-    }
-
-    # The text to split is exported from the database by the IsisGdl program
-    open(FILE, "IsisGdl \"$filename\" |");
-
-    my $reader = new multiread();
-    $reader->set_handle('ISISPlugin::FILE');
-    $reader->set_encoding($encoding);
-    $reader->read_file($textref);
-
-    close(FILE);
-
-    # Parse the associated ISIS database Field Definition Table file (.fdt)
-    my %fdt_mapping = &parse_field_definition_table($self->{'fdt_file_path'}, $encoding);
-    $self->{'fdt_mapping'} = \%fdt_mapping;
-
-    # Remove the line at the start, and any blank lines, so the data is split and processed properly
-    $$textref =~ s/^----------\n//;
-    $$textref =~ s/\n\n/\n/g;
-}
-
-
-sub process
-{
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # store the auxiliary files so we know which ones were used
-    # (mst file becomes the source file)
-    $doc_obj->associate_source_file($self->{'fdt_file_path'});
-    $doc_obj->associate_source_file($self->{'xrf_file_path'});
-
-    my $section = $doc_obj->get_top_section();
-    my $fdt_mapping = $self->{'fdt_mapping'};
-    my $subfield_separator = $self->{'subfield_separator'};
-    my $entry_separator = $self->{'entry_separator'};
-    my $isis_record_html_metadata_value = "<table cellpadding=\"4\" cellspacing=\"0\">";
-
-    # Process each line of the ISIS record, one at a time
-    foreach my $line (split(/\n/, $$textref)) {
-	$line =~ s/(\s*)$//;  # Remove any nasty whitespace (very important for Windows)
-	$line =~ /^tag=(.*) data=(.+)$/;
-	my $tag = $1;
-	my $tag_data = $2;
-        # print STDERR "\nTag: $tag, Data: $tag_data\n";
-
-	# Convert the tag number into a name, and remove any invalid characters
-	my $raw_metadata_name = $fdt_mapping->{$tag}{'name'} || "";
-	$raw_metadata_name =~ s/[,&\#\.\-\/]/ /g;
-	next if ($raw_metadata_name eq "");
-
-	# Metadata field names: title case, then remove spaces
-	my $metadata_name = "";
-	foreach my $word (split(/\s+/, $raw_metadata_name)) {
-	    substr($word, 0, 1) =~ tr/a-z/A-Z/;
-	    $metadata_name .= $word;
-	}
-
-	my $all_metadata_name = $metadata_name . "^all";
-	my $all_metadata_value = "";
-
-	# Handle repeatable fields
-	if ($fdt_mapping->{$tag}{'repeatable'}) {
-	    # Multiple values are separated using the '%' character
-	    foreach my $raw_metadata_value (split(/%/, $tag_data)) {
-		my $metadata_value = "";
-
-		# Handle subfields
-		while ($raw_metadata_value ne "") {
-		    # If there is a subfield specifier, parse it off
-		    my $sub_metadata_name = $metadata_name;
-		    if ($raw_metadata_value =~ s/^\^// && $raw_metadata_value =~ s/^([a-z])//) {
-			$sub_metadata_name .= "^$1";
-		    }
-
-		    # Parse the value off and add it as metadata
-		    $raw_metadata_value =~ s/^([^\^]*)//;
-		    my $sub_metadata_value = &escape_metadata_value($1);
-
-		    # print STDERR "Sub metadata name: $sub_metadata_name, value: $sub_metadata_value\n";
-		    if ($sub_metadata_name ne $metadata_name) {
-			$doc_obj->add_utf8_metadata($section, $sub_metadata_name, $sub_metadata_value); 
-		    }
-
-		    # If this tag has subfields and this is the first, use the value for the CDS/ISIS ^* field
-		    if ($fdt_mapping->{$tag}{'subfields'} ne "" && $metadata_value eq "") {
-			$doc_obj->add_utf8_metadata($section, $metadata_name . "^*", $sub_metadata_value); 
-		    }
-
-		    $metadata_value .= $subfield_separator unless ($metadata_value eq "");
-		    $metadata_value .= $sub_metadata_value;
-		}
-
-		# Add the metadata value
-		# print STDERR "Metadata name: $metadata_name, value: $metadata_value\n";
-		$doc_obj->add_utf8_metadata($section, $metadata_name, $metadata_value); 
-
-		$all_metadata_value .= $entry_separator unless ($all_metadata_value eq "");
-		$all_metadata_value .= $metadata_value;
-	    }
-	}
-
-	# Handle non-repeatable fields
-	else {
-	    my $raw_metadata_value = $tag_data;
-	    my $metadata_value = "";
-
-	    # Handle subfields
-	    while ($raw_metadata_value ne "") {
-		# If there is a subfield specifier, parse it off
-		my $sub_metadata_name = $metadata_name;
-		if ($raw_metadata_value =~ s/^\^// && $raw_metadata_value =~ s/^([a-z])//) {
-		    $sub_metadata_name .= "^$1";
-		}
-
-		# Parse the value off and add it as metadata
-		$raw_metadata_value =~ s/^([^\^]*)//;
-		my $sub_metadata_value = $1;
-
-		# Deal with the case when multiple values are specified using <...>
-		if ($sub_metadata_value =~ /\<(.+)\>/) {
-		    my $sub_sub_metadata_name = $sub_metadata_name . "^sub";
-		    my $tmp_sub_metadata_value = $sub_metadata_value;
-		    while ($tmp_sub_metadata_value =~ s/\<(.+?)\>//) {
-			my $sub_sub_metadata_value = $1;
-			$doc_obj->add_utf8_metadata($section, $sub_sub_metadata_name, $sub_sub_metadata_value); 
-		    }
-		}
-		# Deal with the legacy case when multiple values are specified using /.../
-		elsif ($sub_metadata_value =~ /\/(.+)\//) {
-		    my $sub_sub_metadata_name = $sub_metadata_name . "^sub";
-		    my $tmp_sub_metadata_value = $sub_metadata_value;
-		    while ($tmp_sub_metadata_value =~ s/\/(.+?)\///) {
-			my $sub_sub_metadata_value = $1;
-			$doc_obj->add_utf8_metadata($section, $sub_sub_metadata_name, $sub_sub_metadata_value); 
-		    }
-		}
-
-		# Escape the metadata value so it appears correctly in the final collection
-		$sub_metadata_value = &escape_metadata_value($sub_metadata_value);
-
-		# print STDERR "Sub metadata name: $sub_metadata_name, value: $sub_metadata_value\n";
-		if ($sub_metadata_name ne $metadata_name) {
-		    $doc_obj->add_utf8_metadata($section, $sub_metadata_name, $sub_metadata_value); 
-		}
-
-		# If this tag has subfields and this is the first, use the value for the CDS/ISIS ^* field
-		if ($fdt_mapping->{$tag}{'subfields'} ne "" && $metadata_value eq "") {
-		    $doc_obj->add_utf8_metadata($section, $metadata_name . "^*", $sub_metadata_value); 
-		}
-
-		$metadata_value .= $subfield_separator unless ($metadata_value eq "");
-		$metadata_value .= $sub_metadata_value;
-	    }
-
-	    # Add the metadata value
-	    # print STDERR "Metadata name: $metadata_name, value: $metadata_value\n";
-	    $doc_obj->add_utf8_metadata($section, $metadata_name, $metadata_value); 
-
-	    $all_metadata_value .= $entry_separator unless ($all_metadata_value eq "");
-	    $all_metadata_value .= $metadata_value;
-	}
-
-	# Add the "^all" metadata value
-	# print STDERR "All metadata name: $all_metadata_name, value: $all_metadata_value\n";
-	$doc_obj->add_utf8_metadata($section, $all_metadata_name, $all_metadata_value); 
-
-	$isis_record_html_metadata_value .= "<tr><td valign=top><nobr><b>" . $fdt_mapping->{$tag}{'name'} . "</b></nobr></td><td valign=top>" . $all_metadata_value . "</td></tr>";
-    }
-
-    # Add a reasonably formatted HTML table view of the record as the document text
-    $isis_record_html_metadata_value .= "</table>";
-    $doc_obj->add_utf8_text($section, $isis_record_html_metadata_value);
-
-    # Add the full raw record as metadata
-    my $isis_raw_record_metadata_value = &escape_metadata_value($$textref);
-    $doc_obj->add_utf8_metadata($section, "ISISRawRecord", $isis_raw_record_metadata_value);
-
-    # Add FileFormat metadata
-    $doc_obj->add_utf8_metadata($section, "FileFormat", "CDS/ISIS");
-
-    # Record was processed successfully
-    return 1;
-}
-
-
-sub parse_field_definition_table
-{
-    my $fdtfilename = shift(@_);
-    my $encoding = shift(@_);
-
-    my %fdtmapping = ();
-
-    open(FDT_FILE, "<$fdtfilename") || die "Error: Could not open file $fdtfilename.\n";
-
-    my $fdtfiletext = "";
-    my $reader = new multiread();
-    $reader->set_handle('ISISPlugin::FDT_FILE');
-    $reader->set_encoding($encoding);
-    $reader->read_file($fdtfiletext);
-
-    my $amongstdefinitions = 0;
-    foreach my $fdtfileline (split(/\n/, $$fdtfiletext)) {
-	$fdtfileline =~ s/(\s*)$//;  # Remove any nasty spaces at the end of the lines
-
-	if ($amongstdefinitions) {
-	    my $fieldname      = &unicode::substr($fdtfileline,  0, 30);
-	    my $fieldsubfields = &unicode::substr($fdtfileline, 30, 20);
-	    my $fieldspecs     = &unicode::substr($fdtfileline, 50, 50);
-
-	    # Remove extra spaces
-	    $fieldname =~ s/(\s*)$//;
-	    $fieldsubfields =~ s/(\s*)$//;
-	    $fieldspecs =~ s/(\s*)$//;
-
-	    # Map from tag number to metadata field title, subfields, and repeatability
-	    my $fieldtag = (split(/ /, $fieldspecs))[0];
-	    my $fieldrepeatable = (split(/ /, $fieldspecs))[3];
-	    $fdtmapping{$fieldtag} = { 'name' => $fieldname,
-				       'subfields' => $fieldsubfields,
-				       'repeatable' => $fieldrepeatable };
-	}
-	elsif ($fdtfileline eq "***") {
-	    $amongstdefinitions = 1;
-	}
-    }
-
-    close(FDT_FILE);
-
-    return %fdtmapping;
-}
-
-
-sub escape_metadata_value
-{
-    my $value = shift(@_);
-    $value =~ s/\</&lt;/g;
-    $value =~ s/\>/&gt;/g;
-    $value =~ s/\\/\\\\/g;
-    return $value;
-}
-
-
-sub clean_up_after_exploding
-{
-    my $self = shift(@_);
-
-    # Delete the FDT and XRF files too
-    &util::rm($self->{'fdt_file_path'});
-    &util::rm($self->{'xrf_file_path'});
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ImagePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ImagePlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,174 +1,0 @@
-###########################################################################
-#
-# ImagePlugin.pm -- for processing standalone images
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ImagePlugin;
-
-use BasePlugin;
-use ImageConverter;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-use gsprintf 'gsprintf';
-
-sub BEGIN {
-    @ImagePlugin::ISA = ('BasePlugin', 'ImageConverter');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      ];
-
-my $options = { 'name'     => "ImagePlugin",
-		'desc'     => "{ImagePlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    
-    new ImageConverter($pluginlist, $inputargs, $hashArgOptLists);
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    $self->SUPER::init(@_);
-    $self->ImageConverter::init();
-    $self->{'cover_image'} = 0; # makes no sense for images
-}
-
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    $self->SUPER::begin(@_);
-    $self->ImageConverter::begin(@_);
-}
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-	# from .jpf and onwards below, the file extensions are for JPEG2000
-    return q^(?i)\.(jpe?g|gif|png|bmp|xbm|tif?f|jpf|jpx|jp2|jpc|j2k|pnm|pgx)$^; 
-}
-
-# this makes no sense for images
-sub block_cover_image
-{
-    my $self =shift (@_);
-    my ($filename) = @_;
-
-    return;
-}
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    # options??
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-	
-    if ($self->{'image_conversion_available'} == 1)
-    {
-		my $plugin_filename_encoding = $self->{'filename_encoding'};
-		my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-		
-		my $url_encoded_full_filename 
-		    = &unicode::raw_filename_to_url_encoded($filename_full_path);
-
-		# should we check the return value?
-		$self->generate_images($filename_full_path, 
-				       $url_encoded_full_filename, 
-				       $doc_obj, $doc_obj->get_top_section(),$filename_encoding); 
-		
-    }
-    else
-    {
-		if ($gli) {
-			&gsprintf(STDERR, "<Warning p='ImagePlugin' r='{ImageConverter.noconversionavailable}: {ImageConverter.".$self->{'no_image_conversion_reason'}."}'>");
-		}
-		# all we do is add the original image as an associated file, and set up srclink etc
-		my $assoc_file = $doc_obj->get_assocfile_from_sourcefile();
-		my $section = $doc_obj->get_top_section();
-		
-		$doc_obj->associate_file($filename_full_path, $assoc_file, "", $section);
-		
-		# srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-		$doc_obj->add_metadata ($section, "srclink_file", $doc_obj->get_sourcefile()); 
-		$doc_obj->add_metadata ($section, "srclinkFile", $doc_obj->get_sourcefile()); 
-		# We don't know the size of the image, but the browser should display it at full size
-		$doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/[srclinkFile]\">");
-		
-		# Add a fake thumbnail icon with the full-sized image scaled down by the browser
-		$doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/[srclinkFile]\" width=\"" . $self->{'thumbnailsize'} . "\">");
-    }
-    #we have no text - adds dummy text and NoText metadata
-    $self->add_dummy_text($doc_obj, $doc_obj->get_top_section());
-	
-    return 1;
-	
-}
-
-
-sub clean_up_after_doc_obj_processing {
-    my $self = shift(@_);
-    
-    $self->ImageConverter::clean_up_temporary_files();
-}
-
-1;
-
-
-
-
-
-
-
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/LOMPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/LOMPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,717 +1,0 @@
-###########################################################################
-#
-# LOMPlugin.pm -- plugin for import the collection from LOM
-# 
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-### Note this plugin currently can't download source documents from outside if you are behind a firewall.
-# Unless, you set the http_proxy environment variable to be your proxy server, 
-# and set proxy_user and proxy_password in .wgetrc file in home directory. 
-# (does that work on windows??)
-
-package LOMPlugin;
-
-use ReadTextFile;
-use MetadataPass;
-use MetadataRead;
-use XMLParser;
-use Cwd;
-
-# methods with identical signatures take precedence in the order given in the ISA list.
-sub BEGIN {
-    @ISA = ('MetadataRead', 'ReadTextFile', 'MetadataPass');
-}
-
-use strict; # every perl program should have this!
-no strict 'refs'; # make an exception so we can use variables as filehandles
-
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "string",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "root_tag",
-	'desc' => "{LOMPlugin.root_tag}",
-	'type' => "regexp",
-	'deft' => q/^(?i)lom$/,
-	'reqd' => "no" },
-      { 'name' => "check_timestamp",
-	'desc' => "{LOMPlugin.check_timestamp}",
-	'type' => "flag" },
-      { 'name' => "download_srcdocs",
-	'desc' => "{LOMPlugin.download_srcdocs}",
-	'type' => "regexp",
-	'deft' => "",
-	'reqd' => "no" }];
-
-my $options = { 'name'     => "LOMPlugin",
-		'desc'     => "{LOMPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-
-my ($self);
-sub new {
-    my $class = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-   
-    $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about creating the XML parser as all we want is the 
-	# list of plugin options
-	return bless $self, $class;
-    }
-
-    #create XML::Parser object for parsing dublin_core.xml files
-    my $parser = new XML::Parser('Style' => 'Stream',
-				 'Handlers' => {'Char' => \&Char,
-						'Doctype' => \&Doctype
-						});
-    $self->{'parser'} = $parser;
-
-    $self->{'extra_blocks'} = {};
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.xml$^;
-}
-
-
-sub can_process_this_file {
-    my $self = shift(@_);
-    my ($filename) = @_;
-
-    if ($self->SUPER::can_process_this_file($filename) && $self->check_doctype($filename)) {
-	return 1; # its a file for us
-    }
-    return 0;
-}
-
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
-
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-    
-    print $outhandle "LOMPlugin: extracting metadata from $file\n"
-	if $self->{'verbosity'} > 1;
-
-    my ($dir,$tail) = $filename_full_path =~ /^(.*?)([^\/\\]*)$/;
-    $self->{'output_dir'} = $dir;
-
-    eval {
-	$self->{'parser'}->parsefile($filename_full_path);
-    };
-    
-    if ($@) {
-	print $outhandle "LOMPlugin: skipping $filename_full_path as not conformant to LOM syntax\n" if ($self->{'verbosity'} > 1);
-	print $outhandle "\n Perl Error:\n $@\n" if ($self->{'verbosity'}>2);
-	return 0;
-    }
-
-    $self->{'output_dir'} = undef;
-
-    my $file_re;
-    my $lom_srcdoc = $self->{'lom_srcdoc'};
-
-    if (defined $lom_srcdoc) {
-	my $dirsep = &util::get_re_dirsep();
-	$lom_srcdoc =~ s/^$base_dir($dirsep)//;
-	$self->{'extra_blocks'}->{$file}++;
-	$file_re = $lom_srcdoc;
-    }
-    else {
-	$file_re = $tail;
-    }
-    $file_re = &util::filename_to_regex($file_re);
-    $self->{'lom_srcdoc'} = undef; # reset for next file to be processed
-
-    push(@$extrametakeys,$file_re);
-    $extrametadata->{$file_re} = $self->{'saved_metadata'};
-    if (defined $lom_srcdoc) {
-	# copied from oaiplugin
-	if (!defined $extrametafile->{$file_re}) {
-	    $extrametafile->{$file_re} = {};
-	}
-	 #maps the file to full path
-	$extrametafile->{$file_re}->{$file} = $filename_full_path;
-    }
-	
-    return 1;
-}
-
-sub check_doctype {
-    $self = shift (@_);
-    
-    my ($filename) = @_;
-    
-    if (open(XMLIN,"<$filename")) {
-	my $doctype = $self->{'root_tag'};
-	## check whether the doctype has the same name as the root element tag
-	while (defined (my $line = <XMLIN>)) {
-	    ## find the root element
-	    if ($line =~ /<([\w\d:]+)[\s>]/){
-		my $root = $1;
-		if ($root !~ $doctype){
-		    close(XMLIN);
-		    return 0;
-		}
-		else {
-		    close(XMLIN); 
-		    return 1;
-		}
-	    }
-	}
-	close(XMLIN);
-    }
-    
-    return undef; # haven't found a valid line
-    
-}
-
-sub read_file {
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-
-    my $metadata_table = $self->{'metadata_table'};
-
-    my $rawtext = $metadata_table->{'rawtext'};
-
-    delete $metadata_table->{'rawtext'};
-
-    $$textref = $rawtext;
-}
-
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    return 0 if (defined $self->{'extra_blocks'}->{$file});
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-
-    $self->{'metadata_table'} = $metadata;
-
-    my $lom_language = $metadata->{'lom_language'};
-
-    my $store_input_encoding;
-    my $store_extract_language;
-    my $store_default_language;
-    my $store_default_encoding;
-
-    if (defined $lom_language) {
-	delete $metadata->{'lom_language'};
-
-	$store_input_encoding   = $self->{'input_encoding'};
-	$store_extract_language = $self->{'extract_language'};
-	$store_default_language = $self->{'default_language'};
-	$store_default_encoding = $self->{'default_encoding'};
-
-	$self->{'input_encoding'}   = "utf8";
-	$self->{'extract_language'} = 0;
-	$self->{'default_language'} = $lom_language;
-	$self->{'default_encoding'} = "utf8";
-    }
-
-    my $rv = $self->SUPER::read(@_);
-
-    if (defined $lom_language) {	
-	$self->{'input_encoding'}   = $store_input_encoding;
-	$self->{'extract_language'} = $store_extract_language;
-	$self->{'default_language'} = $store_default_language;
-	$self->{'default_encoding'} = $store_default_encoding;
-    }
-
-    $self->{'metadata_table'} = undef;
-
-    return $rv;
-}
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $cursection = $doc_obj->get_top_section();
-    $doc_obj->add_utf8_text($cursection, $$textref);
-
-    return 1;
-}
-
-sub Doctype {
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    my $root_tag = $self->{'root_tag'};
-
-    if ($name !~ /$root_tag/) {	
-	die "Root tag $name does not match regular expression $root_tag";
-    }
-}
-
-sub StartTag {
-    my ($expat, $element) = @_;
-
-    my %attr = %_;
-    
-    my $raw_tag = "&lt;$element";
-    map { $raw_tag .= " $_=\"$attr{$_}\""; } keys %attr;
-    $raw_tag .= "&gt;";
-
-    if ($element =~ m/$self->{'root_tag'}/) {
-	$self->{'raw_text'} = $raw_tag;
-
-	$self->{'saved_metadata'} = {};
-	$self->{'metaname_stack'} = [];
-	$self->{'lom_datatype'} = "";
-	$self->{'lom_language'} = undef;
-	$self->{'metadatatext'} = "<table class=\"metadata\" width=\"_pagewidth_\" >\n";
-    }
-    else {
-	my $xml_depth = scalar(@{$self->{'metaname_stack'}});
-	$self->{'raw_text'} .= "\n"; 
-	$self->{'raw_text'} .= "&nbsp;&nbsp;" x $xml_depth; 
-	$self->{'raw_text'} .= $raw_tag;
-
-	my $metaname_stack = $self->{'metaname_stack'};
-	push(@$metaname_stack,$element);
-	if (scalar(@$metaname_stack)==1) {
-	    # top level LOM category
-	    my $style = "class=\"metadata\"";
-	    my $open_close
-		= "<a id=\"${element}opencloselink\" href=\"javascript:hideTBodyArea('$element')\">\n";
-	    $open_close
-		.= "<img id=\"${element}openclose\" border=\"0\" src=\"_httpopenmdicon_\"></a>\n";
-
-	    my $header_line = "  <tr $style ><th $style colspan=\"3\">$open_close \u$element</th></tr>\n";
-	    my $md_tbody = "<tbody id=\"$element\">\n";
-
-	    $self->{'mdheader'}     = $header_line;
-	    $self->{'mdtbody'}      = $md_tbody;
-	    $self->{'mdtbody_text'} = "";
-	}
-    }
-}
-
-sub EndTag {
-    my ($expat, $element) = @_;
-
-    my $raw_tag = "&lt;/$element&gt;";
-    
-    if ($element =~ m/$self->{'root_tag'}/) {
-	$self->{'raw_text'} .= $raw_tag;
-
-	my $metadatatext = $self->{'metadatatext'};
-	$metadatatext .= "</table>";
-
-	my $raw_text = $self->{'raw_text'};
-
-	$self->{'saved_metadata'}->{'MetadataTable'} =  $metadatatext;
-	$self->{'metadatatext'} = "";
-
-	$self->{'saved_metadata'}->{'rawtext'} =  $raw_text;
-	$self->{'raw_text'} = "";
-
-	if (defined $self->{'lom_language'}) {
-	    $self->{'saved_metadata'}->{'lom_language'} = $self->{'lom_language'};
-	    $self->{'lom_language'} = undef;
-	}
-    }
-    else {
-	my $metaname_stack = $self->{'metaname_stack'};
-
-	if (scalar(@$metaname_stack)==1) {
-	    my $header_line = $self->{'mdheader'};
-	    my $tbody_start = $self->{'mdtbody'};
-	    my $tbody_text  = $self->{'mdtbody_text'};
-	    if ($tbody_text !~ m/^\s*$/s) {
-		my $tbody_end = "</tbody>\n";
-		my $table_chunk 
-		    = $header_line.$tbody_start.$tbody_text.$tbody_end;
-
-		$self->{'metadatatext'} .= $table_chunk;
-	    }
-	    $self->{'mdtheader'}    = "";
-	    $self->{'mdtbody'}      = "";
-	    $self->{'mdtbody_text'} = "";
-	}
-
-	pop(@$metaname_stack);
-
-	my $xml_depth = scalar(@{$self->{'metaname_stack'}});
-	$self->{'raw_text'} .= "\n"; 
-	$self->{'raw_text'} .= "&nbsp;&nbsp;" x $xml_depth; 
-	$self->{'raw_text'} .= $raw_tag;
-    }
-}
-
-sub process_datatype_info
-{
-    my $self = shift(@_);
-    my ($metaname_stack,$md_content) = @_;
-
-    my @without_dt_stack = @$metaname_stack; # without datatype stack
-
-    my $innermost_element = $without_dt_stack[$#without_dt_stack];
-
-    # Loose last item if encoding datatype information
-    if ($innermost_element =~ m/^(lang)?string$/) {
-	$self->{'lom_datatype'} = $innermost_element;
-
-	pop @without_dt_stack;
-	$innermost_element = $without_dt_stack[$#without_dt_stack];
-    }
-    elsif ($innermost_element =~ m/^date(Time)?$/i) { 
-	if ($innermost_element =~ m/^date$/i) {
-	    $self->{'lom_datatype'} = "dateTime";
-	}
-	else {
-	    $self->{'lom_datatype'} = $innermost_element;
-
-	    pop @without_dt_stack;
-	    $innermost_element = $without_dt_stack[$#without_dt_stack];
-	}
-
-	if ($md_content =~ m/^(\d{1,2})\s*(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\s*(\d{4})/i) {
-	    my ($day,$mon,$year) = ($1,$2,$3);
-	    
-	    my %month_lookup = ( 'jan' =>  1, 'feb' =>  2, 'mar' =>  3,
-				 'apr' =>  4, 'may' =>  5, 'jun' =>  6,
-				 'jul' =>  7, 'aug' =>  8, 'sep' =>  9,
-				 'oct' => 10, 'nov' => 11, 'dec' => 12 );
-
-	    my $mon_num = $month_lookup{lc($mon)};
-
-	    $md_content = sprintf("%d%02d%02d",$year,$mon_num,$day);
-	}
-
-	$md_content =~ s/\-//g;
-    }
-
-    if ($innermost_element eq "source") {
-	$self->{'lom_source'} = $md_content;
-    }
-    elsif ($innermost_element eq "value") {
-	$self->{'lom_value'} = $md_content;
-    }
-
-    return (\@without_dt_stack,$innermost_element,$md_content);
-}
-
-sub reset_datatype_info
-{
-    my $self = shift(@_);
-
-    $self->{'lom_datatype'} = "";
-}
-
-
-sub pretty_print_text
-{
-    my $self = shift(@_);
-    
-    my ($pretty_print_text) = @_;
-
-##    $metavalue_utf8 = &util::hyperlink_text($metavalue_utf8);
-    $pretty_print_text = &util::hyperlink_text($pretty_print_text);
-	
-####    $pretty_print_text =~ s/(BEGIN:vCard.*END:vCard)/<pre>$1<\/pre>/sg;
-
-    if ($self->{'lom_datatype'} eq "dateTime") {
-	if ($pretty_print_text =~ m/^(\d{4})(\d{2})(\d{2})$/) {
-	    $pretty_print_text = "$1-$2-$3";
-	}
-    }
-
-    return $pretty_print_text;
-}
-
-sub pretty_print_table_tr
-{
-    my $self = shift (@_);
-    my ($without_dt_stack) = @_;
-
-    my $style = "class=\"metadata\"";
-
-    my $innermost_element = $without_dt_stack->[scalar(@$without_dt_stack)-1];
-    my $outermost_element = $without_dt_stack->[0];
-
-    # Loose top level stack item (already named in pretty print table)
-    my @pretty_print_stack = @$without_dt_stack;
-    shift @pretty_print_stack; 
-
-    if ($innermost_element eq "source") {
-	return if (!defined $self->{'lom_value'});
-    }
-
-    if ($innermost_element eq "value") {
-	return if (!defined $self->{'lom_source'});
-    }
-
-    my $pretty_print_text = "";
-
-    if (($innermost_element eq "value") || ($innermost_element eq "source")) {
-	my $source = $self->{'lom_source'};
-	my $value  = $self->pretty_print_text($self->{'lom_value'});
-
-	$self->{'lom_source'} = undef;
-	$self->{'lom_value'} = undef;
-
-	pop @pretty_print_stack;
-	
-	$pretty_print_text = "<td $style>$source</td><td $style>$value</td>";
-    }
-    else {
-	$pretty_print_text = $self->pretty_print_text($_);
-	$pretty_print_text = "<td $style colspan=2>$pretty_print_text</td>";
-    }
-    my $pretty_print_fmn = join(' : ',map { "\u$_"; } @pretty_print_stack);
-
-
-    # my $tr_attr = "id=\"$outermost_element\" style=\"display:block;\"";
-    my $tr_attr = "$style id=\"$outermost_element\"";
-
-    my $mdtext_line = "  <tr $tr_attr><td $style><nobr>$pretty_print_fmn</nobr></td>$pretty_print_text</tr>\n";
-    $self->{'mdtbody_text'} .= $mdtext_line;
-}
-
-
-sub check_for_language
-{
-    my $self = shift(@_);
-    my ($innermost_element,$md_content) = @_;
-
-    # Look for 'language' tag
-    if ($innermost_element eq "language") {
-	my $lom_lang = $self->{'lom_language'};
-	
-	if (defined $lom_lang) {
-	    my $new_lom_lang = $md_content;
-	    $new_lom_lang =~ s/-.*//; # remove endings like -US or -GB
-
-	    if ($lom_lang ne $new_lom_lang) {
-		my $outhandle = $self->{'outhandle'};
-		
-		print $outhandle "Warning: Conflicting general language in record\n";
-		print $outhandle "         $new_lom_lang (previous value for language = $lom_lang)\n";
-	    }
-	    # otherwise, existing value OK => do nothing
-	}
-	else {
-	    $lom_lang = $md_content;
-	    $lom_lang =~ s/-.*//; # remove endings like -US or -GB
-	    
-	    $self->{'lom_language'} = $lom_lang;
-	}
-    }
-}
-
-sub found_specific_identifier
-{
-    my $self = shift(@_);
-    my ($specific_id,$full_mname,$md_content) = @_;
-
-    my $found_id = 0;
-    if ($full_mname eq $specific_id) {
-	if ($md_content =~ m/^(http|ftp):/) {
-	    $found_id = 1;
-	}
-    }
-
-    return $found_id;
-}
-
-sub download_srcdoc
-{
-    my $self = shift(@_);
-    my ($doc_url) = @_;
-
-    my $outhandle  = $self->{'outhandle'};
-    my $output_dir = $self->{'output_dir'};
-
-    $output_dir = &util::filename_cat($output_dir,"_gsdldown.all");
-
-    if (! -d $output_dir) {
-	mkdir $output_dir;
-    }
-
-    my $re_dirsep = &util::get_re_dirsep();
-    my $os_dirsep = &util::get_dirsep();
-
-    my $file_url = $doc_url;
-    $file_url =~ s/$re_dirsep/$os_dirsep/g;
-    $file_url =~ s/^(http|ftp):\/\///;
-    $file_url .= "index.html" if ($file_url =~ m/\/$/);
-
-    my $full_file_url = &util::filename_cat($output_dir,$file_url);
-    # the path to srcdoc will be used later in extrametadata to associate
-    # the lom metadata with the document. Needs to be relative to current
-    # directory.
-    my $srcdoc_path = &util::filename_cat("_gsdldown.all", $file_url);
-    my $check_timestamp = $self->{'check_timestamp'};
-    my $status;
-
-    if (($check_timestamp) || (!$check_timestamp && !-e $full_file_url)) {
-	if (!-e $full_file_url) {
-	    print $outhandle "Mirroring $doc_url\n";
-	}
-	else {
-	    print $outhandle "Checking to see if update needed for $doc_url\n";
-	}
-
-	# on linux, if we pass an absolute path as -P arg to wget, then it 
-	# stuffs up the 
-	# URL rewriting in the file. Need a relative path or none, so now
-	# we change working directory first.
-	my $changed_dir = 0;
-	my $current_dir = cwd();
-	my $wget_cmd = "";
-	if ($ENV{'GSDLOS'} ne "windows") {
-	    $changed_dir = 1;
-	    
-	    chdir "$output_dir";
-	    $wget_cmd = "wget -nv  --timestamping -k -p \"$doc_url\"";
-	} else {
-	    $wget_cmd = "wget -nv -P \"$output_dir\" --timestamping -k -p \"$doc_url\""; 
-	}
-	##print STDERR "**** wget = $wget_cmd\n";
-
-	
-	$status = system($wget_cmd);
-	if ($changed_dir) {
-	    chdir $current_dir;
-	}
-	if ($status==0) {
-	    $self->{'lom_srcdoc'} = $srcdoc_path; 	
-	}
-	else {
-	    $self->{'lom_srcdoc'} = undef;
-	    print $outhandle "Error: failed to execute $wget_cmd\n";
-	}
-    }
-    else {
-	# not time-stamping and file already exists
-	$status=0;
-	$self->{'lom_srcdoc'} = $srcdoc_path; 	
-    }
-
-    return $status==0;
-    
-}
-
-
-sub check_for_identifier
-{
-    my $self = shift(@_);
-    my ($full_mname,$md_content) = @_;
-
-    my $success = 0;
-
-    my $download_re = $self->{'download_srcdocs'};
-    if (($download_re ne "") && $md_content =~ m/$download_re/) {
-	
-	if ($self->found_specific_identifier("general^identifier^entry",$full_mname,$md_content)) {
-	    $success = $self->download_srcdoc($md_content);
-	}
-
-	if (!$success) {
-	    if ($self->found_specific_identifier("technical^location",$full_mname,$md_content)) {
-		$success = $self->download_srcdoc($md_content);
-	    }
-	}
-    }
-
-    return $success;
-}
-
-
-sub Text {
-    if ($_ !~ m/^\s*$/) {
-	#
-	# Work out indentations and line wraps for raw XML
-	#
-	my $xml_depth = scalar(@{$self->{'metaname_stack'}})+1;
-	my $indent = "&nbsp;&nbsp;" x $xml_depth; 
-	
-	my $formatted_text = "\n".$_;
-
-	# break into lines < 80 chars on space
-	$formatted_text =~ s/(.{50,80})\s+/$1\n/mg; 
-	$formatted_text =~ s/^/$indent/mg;
-	## $formatted_text =~ s/\s+$//s;
-
-	$self->{'raw_text'} .= $formatted_text;
-    }
-
-    my $metaname_stack = $self->{'metaname_stack'};
-    if (($_ !~ /^\s*$/) && (scalar(@$metaname_stack)>0)) {
-
-	my ($without_dt_stack,$innermost_element,$md_content)
-	    = $self->process_datatype_info($metaname_stack,$_);
-
-	$self->pretty_print_table_tr($without_dt_stack);
-
-	my $full_mname = join('^',@{$without_dt_stack});
-	$self->set_filere_metadata(lc($full_mname),$md_content);
-
-	$self->check_for_language($innermost_element,$md_content);
-	$self->check_for_identifier($full_mname,$md_content); # source doc
-
-	$self->reset_datatype_info();
-    }
-}
-
-# This Char function overrides the one in XML::Parser::Stream to overcome a
-# problem where $expat->{Text} is treated as the return value, slowing
-# things down significantly in some cases.
-sub Char {
-  $_[0]->{'Text'} .= $_[1];
-  return undef;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MARCXMLPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MARCXMLPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,524 +1,0 @@
-###########################################################################
-#
-# MARCXMLPlugin.pm
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Processes MARCXML documents. Note that this plugin does no
-# syntax checking (though the XML::Parser module tests for
-# well-formedness).
-
-package MARCXMLPlugin;
-
-use ReadXMLFile;
-use ReadTextFile;
-use MetadataRead;
-use marcmapping;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-# methods with identical signatures take precedence in the order given in the ISA list.
-sub BEGIN {
-    @MARCXMLPlugin::ISA = ('MetadataRead', 'ReadXMLFile', 'ReadTextFile');
-}
-
-my $arguments = [{'name' => "metadata_mapping_file",
-		  'desc' => "{MARCXMLPlugin.metadata_mapping_file}",
-		  'type' => "string",
-		  'deft' => "marc2dc.txt",
-		  'reqd' => "no" },
-		 { 'name' => "process_exp",
-		   'desc' => "{BasePlugin.process_exp}",
-		   'type' => "regexp",
-		   'deft' => &get_default_process_exp(),
-		   'reqd' => "no" }];
-
-my $options = { 'name'     => "MARCXMLPlugin",
-		'desc'     => "{MARCXMLPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments 
-		};
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    # we want to be able to use the textcat methods from ReadTextFile
-    # to get the language and encoding
-    new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists, 1);
-
-    my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-        
-    # we want to strip namespaces, so have to create a new XML parser
-    my $parser = new XML::Parser('Style' => 'Stream',
-                                 'Pkg' => 'ReadXMLFile',
-                                 'PluginObj' => $self,
-				 'Namespaces' => 1, # strip out namespaces
-				 'Handlers' => {'Char' => \&ReadXMLFile::Char,
-						'XMLDecl' => \&ReadXMLFile::XMLDecl,
-						'Entity'  => \&ReadXMLFile::Entity,
-						'Doctype' => \&ReadXMLFile::Doctype,
-						'Default' => \&ReadXMLFile::Default
-                                 });
-
-    $self->{'parser'} = $parser;
-
-    $self->{'content'} = "";
-    $self->{'xmlcontent'} = "";
-    $self->{'record_count'} = 1;
-    $self->{'language'} = "";
-    $self->{'encoding'} = "";
-    $self->{'marc_mapping'} = {};
-    $self->{'current_code'} = "";
-    $self->{'current_tag'} = "";
-    $self->{'current_element'} = "";
-    $self->{'metadata_mapping'} = undef;
-    $self->{'num_processed'} = 0;
-    $self->{'indent'} = 0;
-
-    return bless $self, $class;
-}
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.xml$^;
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "collection";
-}
-
-
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-    
-    ## the mapping file has already been loaded
-    if (defined $self->{'metadata_mapping'} ){ 
-	$self->SUPER::init(@_);
-	return;
-    }
-
-    # read in the metadata mapping file
-    my $mm_file = &util::locate_config_file($self->{'metadata_mapping_file'}); 
-
-    if (! defined $mm_file)
-    {
-	my $msg = "MARCXMLPlugin ERROR: Can't locate mapping file \"" .
-	    $self->{'metadata_mapping_file'} . "\".\n " .
-		"    No metadata will be extracted from MARCXML files.\n";
-
-	print $outhandle $msg;
-	print $failhandle $msg;
-	$self->{'metadata_mapping'} = undef;
-	# We pick up the error in process() if there is no $mm_file
-	# If we exit here, then pluginfo.pl will exit too!
-    }
-    else {
-	$self->{'metadata_mapping'} = &marcmapping::parse_marc_metadata_mapping($mm_file, $outhandle);
-    }
-
-
-    ##map { print STDERR $_."=>".$self->{'metadata_mapping'}->{$_}."\n"; } keys %{$self->{'metadata_mapping'}};
-
-    $self->SUPER::init(@_);
-}
-
-# Called for DOCTYPE declarations - use die to bail out if this doctype
-# is not meant for this plugin
-sub xml_doctype {
-    my $self = shift(@_);
-
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-   return;
-
-}
-
-
-sub xml_start_document {
-    my $self = shift(@_);
-
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-      
-    my $file = $self->{'file'};
-    my $filename = $self->{'filename'};
-       
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
-
-    $self->{'language'} = $language;
-    $self->{'encoding'} = $encoding;
-    $self->{'element_count'} = 1;
-    $self->{'indent'} = 0;
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "MARCXMLPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
-    print STDERR "<Processing n='$self->{'file'}' p='MARCXMLPlugin'>\n" if $self->{'gli'};
-
-    # reset the base id
-    $self->{'base_oid'} = undef;
- 
-}
-
-sub xml_end_document {
-
-}
-
-sub xml_start_tag {
-    my $self = shift;
-    my $expat = shift;
-    my $element = shift;  
-
-    my $text = $_;
-    my $escaped_text =  $self->escape_text($_); 
-  
-    $self->{'current_element'} = $element;
-
-    ##get all atributes of this element and store it in a map name=>value    
-    my %attr_map = (); 
-    my $attrstring = $_;
-    while ($attrstring =~ /(\w+)=\"(\w+)\"/){
-	$attr_map{$1}=$2;
-	$attrstring = $'; #'
-    }
-
-
-    my $processor = $self->{'processor'};
-	my $metadata  = $self->{'metadata'};
-
-    ##create a new document for each record 
-    if ($element eq "record") {
-        my $filename = $self->{'filename'};
-	my $language = $self->{'language'};
-        my $encoding = $self->{'encoding'};
-	my $file = $self->{'file'};
-	my $doc_obj = new doc($filename, undef, $self->{'file_rename_method'});
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Language", $language);
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Encoding", $encoding);
-
-	my ($filemeta) = $file =~ /([^\\\/]+)$/;
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-	$self->set_Source_metadata($doc_obj, $filename, $filename_encoding);
-
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "SourceSegment", "$self->{'record_count'}");
-        if ($self->{'cover_image'}) {
-	    $self->associate_cover_image($doc_obj, $filename);
-	}
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "MARCXML");
-
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "Record $self->{'record_count'}\n" if $self->{'verbosity'} > 1; 
-
-        $self->{'record_count'}++;
-        $self->{'doc_obj'} = $doc_obj;       
-	$self->{'num_processed'}++;
-	if (!defined $self->{'base_oid'}) {
-	    $self->SUPER::add_OID($doc_obj);
-	    $self->{'base_oid'} = $doc_obj->get_OID();
-	}
-	
-
-    }
-    
-    ## get the marc code, for example 520
-     if ($element eq "datafield") {
-    	 if (defined $attr_map{'tag'} and $attr_map{'tag'} ne ""){
-	     $self->{'current_tag'} = $attr_map{tag};  
-	 }
-     }
-
-
-    ## append the subcode to the marc code for example 520a or 520b 
-    if ($element eq "subfield"){
-   	if (defined $attr_map{'code'} and $attr_map{'code'} ne "" and $self->{'current_tag'} ne ""){
-	    $self->{'current_code'} = $attr_map{'code'};
-	}
-    }
-
-   if ($element eq "record"){
-        $self->{'indent'} = 0;
-        $self->{'content'} = "";
-        $self->{'xmlcontent'} = "";
-    }
-    else {
-         if ($element ne "subfield"){
-              $self->{'indent'} = 1;
-         }
-         else{
-           $self->{'indent'} = 2;
-         }
-    }
-    
-
-    if ($element eq "collection") {
-	# remember the full start tag for <collection ...>
-	# This is needed to wrap around each <record> when generating its associate MARCXML file
-
-        $self->{'xmlcollectiontag'} = $text;
-    }
-    else {
-        $self->{'content'} .= "<br/>" if ($element ne "record");
-        $self->{'content'} .= $self->calculate_indent($self->{'indent'}).$escaped_text;
-        $self->{'xmlcontent'} .= $text;
-   }
-    
-}
-
-
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    my $text = $_;
-    my $escaped_text =  $self->escape_text($_); 
- 
-    if ($element eq "record" and defined $self->{'doc_obj'}) {
-	# process the document
-	my $processor = $self->{'processor'};
-	my $doc_obj = $self->{'doc_obj'};
-        $self->{'content'} .= "<br/>".$escaped_text;
-        $self->{'xmlcontent'} .= $text;
-      
-
-	my $top_section = $doc_obj->get_top_section();
-
-	my $tmp_marcxml_filename = &util::get_tmp_filename("xml");
-	if (open (XMLOUT,">$tmp_marcxml_filename")) {
-
-	    print XMLOUT "<?xml-stylesheet type=\"text/xsl\" href=\"MARC21slim2English.xsl\"?>\n";	    
-	    my $xml_content = $self->{'xmlcontent'};
-
-	    $xml_content = $self->{'xmlcollectiontag'}.$xml_content."</collection>";
-
-	    print XMLOUT $xml_content;
-
-	    close(XMLOUT);
-
-	    $doc_obj->associate_file($tmp_marcxml_filename,"marcxml.xml","text/xml", $top_section);
-	    
-	    # assicate xsl style file for presentation as HTML
-	    my $xsl_filename = &util::filename_cat($ENV{'GSDLHOME'},"etc","MARC21slim2English.xsl");
-	    $doc_obj->associate_file($xsl_filename,"MARC21slim2English.xsl","text/xml", $top_section);
-
-	}
-	else {
-	    my $outhandle = $self->{'outhandle'};
-	    print $outhandle "Warning: Unable for write out associated MARCXML file $tmp_marcxml_filename\n";
-	}
-	
-	# include any metadata passed in from previous plugins 
-	# note that this metadata is associated with the top level section
-	
-	$self->extra_metadata ($doc_obj, 
-			       $doc_obj->get_top_section(), 
-			       $self->{'metadata'});
-	
-
-	$self->add_OID($doc_obj, $self->{'base_oid'}, $self->{'record_count'});
-
-	$doc_obj->add_utf8_text($doc_obj->get_top_section(),$self->{'content'});
-        $processor->process($doc_obj);
-
-        ##clean up
-	$self->{'content'} = "";  
-	$self->{'xmlcontent'} = "";  
-	$self->{'doc_obj'} = undef;
-        return;
-    }
-
-    ## map the xmlmarc to gsdl metadata
-    if ($element eq "datafield" and defined $self->{'doc_obj'} and defined $self->{'marc_mapping'} and defined $self->{'metadata_mapping'}){
-	my $metadata_mapping = $self->{'metadata_mapping'};
-	my $marc_mapping = $self->{'marc_mapping'};
-	my $doc_obj = $self->{'doc_obj'};
-
-	##print STDERR "**** Marc Record\n";
-      ##map { print STDERR $_."=>".$marc_mapping->{$_}."\n"; } keys %$marc_mapping;
-	##print STDERR "**** Metadata Mapping\n";
-      ##map { print STDERR $_."=>".$metadata_mapping->{$_}."\n"; } keys %$metadata_mapping;
-
-
-	foreach my $marc_field (keys %$metadata_mapping){
-
-	    ## test whether this field has subfield
-	    my $subfield = undef;
-	    if ($marc_field =~ /(\d\d\d)(?:\$|\^)?(\w)/){
-		$marc_field = $1;
-		$subfield = $2;
-	    }
-
-	    my $matched_field = $marc_mapping->{$marc_field}; 
-
-	    if (defined $matched_field) {
-
-		my $meta_name  = undef;
-		my $meta_value = undef;
-
-		if (defined $subfield){
-		    $meta_name = $metadata_mapping->{$marc_field."\$".$subfield};
-
-		    $meta_value = $matched_field->{$subfield};
-		    
-		    if (!defined $meta_value) {
-			# record read in does not have the specified subfield
-			next;
-		    }
-		}
-		else {
-		    $meta_name = $metadata_mapping->{$marc_field};
-		    
-		    # no subfield => get all the values
-		    my $first = 1;
-		    foreach my $value (sort keys %{$matched_field}) {
-			if ($first) {
-			    $meta_value = $matched_field->{$value};
-			    $first = 0;
-			} else {
-			    $meta_value .= " " . $matched_field->{$value};
-			}
-		    }
-
-		}
-		
-		## escape [ and ]
-		$meta_value =~ s/\[/\\\[/g;
-		$meta_value =~ s/\]/\\\]/g;
-		##print STDERR  "$meta_name=$meta_value\n";
-		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(),$meta_name, $meta_value);
-		
-	    }		    
-			
-	}
-
-	##clean up
-	$self->{'marc_mapping'} = undef;
-	$self->{'current_tag'} = "";
-    }
-  
-   if ($element eq "datafield"){
-       $self->{'indent'} = 1;
-       $self->{'content'} .= "<br/>".$self->calculate_indent($self->{'indent'}).$escaped_text;
-       $self->{'xmlcontent'} .= $text;
-   }
-    else{
-	$self->{'content'} .= $escaped_text;   
-	$self->{'xmlcontent'} .= $text;   
-    }
-     
-}
-
-sub add_OID {
-    my $self = shift (@_);
-    my ($doc_obj, $id, $record_number) = @_;
-
-    my $full_id = $id . "r" . $record_number;
-    if ($self->{'OIDtype'} eq "assigned") {
-	my $identifier = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), $self->{'OIDmetadata'});
-	if (defined $identifier && $identifier ne "") {
-	    $full_id = $identifier;
-	    $full_id = &util::tidy_up_oid($full_id);
-	}
-    }
-    $doc_obj->set_OID($full_id);
-}
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    my $text = $_;
-    my $escaped_text = $self->escape_text($_);
-
-    # protect against & in raw text file
-    $text =~ s/&/&amp;/g; # can't have & in raw form, even in 'raw' xml text
-
-    ## store the text of a marc code, for exapmle 520a=>A poem about....
-    if ($self->{'current_element'} eq "subfield" and $self->{'current_code'} ne "" and $_ ne "" ){
-	##stored it in the marc_mapping 
-
-	my $current_tag  = $self->{'current_tag'};
-	my $current_code = $self->{'current_code'};
-
-     	$self->{'marc_mapping'}->{$current_tag}->{$current_code} .= $_;
-
-	$self->{'current_code'} = "";
-    }
-    
-    $self->{'content'} .= $escaped_text;
-    $self->{'xmlcontent'} .= $text;
-   
-}
-
-sub calculate_indent{
-   my ($self,$num) = @_;
-
-   my $indent ="";
-  
-   for (my $i=0; $i<$num;$i++){
-       $indent .= "&nbsp;&nbsp;&nbsp;&nbsp;";
-    } 
- 
-   return $indent;
-
-}
-
-sub escape_text {
-    my ($self,$text) = @_;
-    # special characters in the xml encoding
-    $text =~ s/&/&amp;/g; # this has to be first...
-    $text =~ s/</&lt;/g;
-    $text =~ s/>/&gt;/g;
-    $text =~ s/\"/&quot;/g;
-
-    return $text;
-}
-
-
-sub unescape_text {
-    my ($self,$text) = @_;
-    # special characters in the xml encoding
-    $text =~ s/&lt;/</g;
-    $text =~ s/&gt;/>/g;
-    $text =~ s/&quot;/\"/g;
-
-    $text =~ s/&/&amp;/g; # can't have & in raw form, even in unescaped xml!
-
-    return $text;
-}
-
-
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MediaWikiPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MediaWikiPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,665 +1,0 @@
-###########################################################################
-#
-# MediaWikiPlugin.pm -- html plugin with extra facilities for wiki page 
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-# This plugin is to process an HTML file from a MediaWiki website which downloaded by 
-# the MediaWikiDownload plug. This plugin will trim MediaWiki functional sections like 
-# login, discussion, history, etc. Only the navigation and search section could be preserved. 
-# Searchbox will be modified to search the Greenstone collection instead of the website.
-# It also can automatically add the table of contents on the website's Main_Page to the 
-# collection's Home page. 
-
-package MediaWikiPlugin;
-
-use HTMLPlugin;
-use unicode;
-
-use strict; # every perl program should have this!
-no strict 'refs'; # make an exception so we can use variables as filehandles
-
-
-sub BEGIN {
-    @MediaWikiPlugin::ISA = ('HTMLPlugin');        
-}
-
-my $arguments = 
-    [          
-     # show the table of contents on collection's home page
-     { 'name' => "show_toc",
-       'desc' => "{MediaWikiPlugin.show_toc}",
-       'type' => "flag",
-       'reqd' => "no"},
-     # set to delete the table of contents section on each MediaWiki page
-     { 'name' => "delete_toc",
-       'desc' => "{MediaWikiPlugin.delete_toc}",
-       'type' => "flag",
-       'reqd' => "no"},
-     # regexp to match the table of contents
-     { 'name' => "toc_exp",
-       'desc' => "{MediaWikiPlugin.toc_exp}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "<table([^>]*)id=(\\\"|')toc(\\\"|')(.|\\n)*?</table>\\n" },        
-     # set to delete the navigation section
-     { 'name' => "delete_nav",
-       'desc' => "{MediaWikiPlugin.delete_nav}",
-       'type' => "flag",
-       'reqd' => "no",
-       'deft' => ""}, 
-     # regexp to match the navigation section    
-     { 'name' => "nav_div_exp",
-       'desc' => "{MediaWikiPlugin.nav_div_exp}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "<div([^>]*)id=(\\\"|')p-navigation(\\\"|')(.|\\n)*?<\/div>" },
-     # set to delete the searchbox section
-     { 'name' => "delete_searchbox",
-       'desc' => "{MediaWikiPlugin.delete_searchbox}",
-       'type' => "flag",
-       'reqd' => "no",
-       'deft' => ""},
-     # regexp to match the searchbox section
-     { 'name' => "searchbox_div_exp",
-       'desc' => "{MediaWikiPlugin.searchbox_div_exp}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "<div([^>]*)id=(\\\"|')p-search(\\\"|')(.|\\n)*?<\/div>"},     
-     # regexp to match title suffix
-     # can't use the title_sub option in HTMLPlugin instead
-     # because title_sub always matches from the begining      
-     { 'name' => "remove_title_suffix_exp",
-       'desc' => "{MediaWikiPlugin.remove_title_suffix_exp}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => ""}
-     ];
-
-my $options = { 'name'     => "MediaWikiPlugin",
-		'desc'     => "{MediaWikiPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $self = new HTMLPlugin($pluginlist, $inputargs, $hashArgOptLists);    
-    return bless $self, $class;
-}
-
-
-
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my @head_and_body = split(/<body/i,$$textref);
-    my $head = shift(@head_and_body);
-    my $body_text = join("<body", @head_and_body);      
-    
-    $head =~ m/<title>(.+)<\/title>/i;
-    my $doctitle = $1 if defined $1;    
-    
-    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'}=~ /\S/) {
-	my @doc_properties = split(/<xml>/i,$head);
-	my $doc_heading = shift(@doc_properties);
-	my $rest_doc_properties = join(" ", @doc_properties);
-	
-	my @extracted_metadata = split(/<\/xml>/i, $rest_doc_properties);
-	my $extracted_metadata = shift (@extracted_metadata);
-	$self->extract_metadata($extracted_metadata, $metadata, $doc_obj);
-    }
-    
-    # set the title here if we haven't found it yet
-    if (!defined $doc_obj->get_metadata_element ($doc_obj->get_top_section(), "Title")) {    
-	if (defined $doctitle && $doctitle =~ /\S/) {        	    
-            # remove suffix in title if required
-            my $remove_suffix_exp = $self->{'remove_title_suffix_exp'};
-	    if (defined $remove_suffix_exp && $remove_suffix_exp =~ /\S/){
-	       $doctitle =~ s/$remove_suffix_exp//i;
-	    }	    
-	    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Title", $doctitle);
-	} else {
-	    $self->title_fallback($doc_obj,$doc_obj->get_top_section(),$file);
-	}
-    }
-
-    # we are only interested in the column-contents div <div id="column-content">
-    # remove header section, it may contain header images or additional search boxes
-    my $header_exp = "<div([^>]*)id=(\"|')container(\"|')([^>]*)>(.|\\n)*<div([^>]*)id=(\"|')column-content";
-    if($body_text =~ /$header_exp/){
-	    $body_text =~ s/$header_exp/<div$1id='container'$4><div$6id='column-content/isg;
-	} else {				
-		$header_exp = "(.|\\n)*?<div([^>]*)?id=(\"|')column-content";
-                if($body_text =~ /$header_exp/){
-                  $body_text =~ s/$header_exp/<div$2id='column-content/i;                
-                }
-	}
-    
-    # remove timeline
-    $body_text =~ s/<div([^>]*)class=("|')smwtimeline("|')[\s\S]*?<\/div>//mg;
-    
-    # remove extra bits
-    my $extra_bits = "Retrieved from(.+)</a>\"";
-    $body_text =~ s/$extra_bits//isg;
-    
-    $body_text =~ s/(<p[^>]*><span[^>]*><o:p>&nbsp;<\/o:p><\/span><\/p>)//isg;
-    $body_text =~ s/(<p[^>]*><o:p>&nbsp;<\/o:p><\/p>)//isg;
-    $body_text =~ s/<!\[if !vml\]>/<![if vml]>/g; 
-    $body_text =~ s/(&nbsp;)+/&nbsp;/sg;
-    
-    # get rid of the [edit] buttons
-    $body_text =~ s/\[<a([^>]*)>edit<\/a>]//g;
-    # get rid of the last time edit information at the bottom
-    $body_text =~ s/<a href="([^>]*)edit([^>]*)"([^>]*?)>(\w+)<\/a> \d\d:\d\d,([\s|\w]*?)\(PST\)//g;    
-    # get rid of the (Redirected from ...)
-    $body_text =~ s/\(Redirected from <a ([^>]*)>(\w|\s)*?<\/a>\)//isg;  
-    
-    # escape texts macros
-    $body_text =~ s/_([^\s]*)_/_<span>$1<\/span>_/isg;
-    # may change the links, like Greenstone_Documentation_All.html, then change back
-    $body_text =~ s/<a([^>]*)_<span>([^>]*)<\/span>_/<a$1_$2_/isg;
-    
-    # define file delimiter for different platforms
-    my $file_delimiter;
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-       $file_delimiter = "\\";
-    } else {
-       $file_delimiter = "/";	        
-    }    
-    
-    # IMPORTANT: different delimiter for $base_dir and $file
-    # $base_dir use forward slash for both windows and linux
-    # print "\nbase_dir : $base_dir\n\n"; # windows: C:/Program Files/Greenstone2.73/collect/wiki/import    
-                                        # linux: /research/lh92/greenstone/greenstone2.73/collect/wiki/import
-    # $file use different delimiters : forward slash for linux; backward slash for windows
-    # print "\nfile : $file\n\n";         # windows: greenstone.sourceforge.net\wiki\index.php\Access_Processing_using_DBPlugin.html    
-                                        # linux: greenstone.sourceforge.net/wiki/index.php/Using_GreenstoneWiki.html
-    
-    # get the base url for the MediaWiki website
-    my $safe_delimiter = &safe_escape_regexp($file_delimiter);
-    my @url_dirs=split($safe_delimiter, $file);
-    my $url_base = $url_dirs[0];    
-        
-    # Re-check css files associated with MediaWiki pages
-    if(defined $base_dir && $base_dir ne ""){ 	
-	my @css_files;
-	my $css_file_count = 0;
-	
-	# find all the stylesheets imported with @import statement	
-	while($head =~ m"<style type=\"text/css\"(.+)import \"(.+)\""ig){
-	    $css_files[$css_file_count++] = $2 if defined $2;
-	}
-	    
-	# download the stylesheets if we haven't downloaded them yet
-        # add prefix to each style elmement, comment out the body element
-        # and copy the files to collection's style folder 
-	for ($css_file_count = 0; $css_file_count < scalar(@css_files); $css_file_count++) {	    
-	    
-	    my $css_file = $css_files[$css_file_count];	      
-	    
-	    # remove prefix gli/cache directory	                
-            $css_file =~ s/^(.+)gli(\\|\/)cache(\\|\/)//i;
-                        
-            # change the \ delimiter in $css_file to / for consistency
-            $css_file =~ s/\\/\//isg;
-            if($css_file !~ /$url_base/) {
-              $css_file = $url_base . $css_file;  
-            }
-            
-            # trim the ? mark append to the end of a stylesheet
-	    $css_file =~ s/\?(.+)$//isg;  
-	   
-            my $css_file_path = &util::filename_cat($base_dir, $css_file);	    
-	    
-	    # do nothing if we have already downloaded the css files
-	    if (! -e $css_file_path) {		
-	     
-             # check the stylesheet's directory in the import folder
-             # if the directory doesn't exist, create one            
-	     my @dirs = split(/\//i,$css_file);	    
-	     my $path_check = "$base_dir/";            
-	     for (my $i = 0; $i < (scalar(@dirs)-1); $i++) {
-		$path_check .= $dirs[$i] . "/";	
-		mkdir($path_check) if (! -d $path_check );
-	     }
-	     
-             # NOTE: wget needs configuration to directly access Internet
-             # These files should already downloaded if we used the MediaWikiDownload             
-	     # downloading            
-	     $css_file = "http://$css_file";	    
-             print "\ndownloading : " . $css_file . "\n\n";
-	     system("wget", "--non-verbose", "$css_file", "--output-document=$css_file_path");
-	     if ($? != 0) {
-              print "[ERROR] Download Failed! Make sure WGet connects to Internet directly \n";
-              print "[ERROR] OR ues the MediaWikiDownload in the GLI DownloadPanel to download from a MediaWiki website\n";
-              unlink("$css_file_path");
-             }
-            } # done with download
-		
-	    # add a prefix "#wikispecificstyle" to each element
-	    # because we want to preserve this website's formats and don't want to mess up with Greenstone formats
-            # so we will wrap the web page with a div with id = wikispecificstyle
-            my $css_content;
-	    if(open(INPUT, "<$css_file_path")){		
-		while(my $line = <INPUT>){
-                    # comment out the body element because we change the body to div
-                    $line =~ s/^(\s*)body(\s*){(\s*)$/$1\/*body$2*\/{$3/isg;
-                    
-                    if($line =~ m/^(.+)\{/i || $line =~ m/^(\s)*#/i){                    
-                      if($line !~ m/wikispecificstyle/i){
-                        $line = "#wikispecificstyle " . $line;
-                      } 
-                    }
-                    
-		    $css_content .= $line;
-		}
-		close(INPUT);			
-		open(OUTPUT, ">$css_file_path");
-		print OUTPUT $css_content;
-		close(OUTPUT);
-	    }
-            
-            # Copy the modified stylesheets to collection's style folder
-            # for future customization
-            my $style_dir = $base_dir;
-            $style_dir =~ s/import$/style/;
-            $css_file =~ m/(.*)\/(.*)$/;
-            $style_dir = &util::filename_cat($style_dir, $2);            
-            
-            if(open(OUTPUT, ">$style_dir")){	
-              print OUTPUT $css_content;
-              close(OUTPUT);
-            }
-	}
-    }    
-    
-    
-    # by default, only preserve navigation box and search box
-    # others like toolbox, interaction, languages box, will be removed  
-    
-    # extract the larger part -- footer section
-    my $print_footer = "<div class=\"printfooter\">(.|\n)+</body>";
-    $body_text =~ /$print_footer/;
-    my $footer = "";
-    $footer = $& if defined $&;
-    $footer =~ s/<\/body>//isg;
-    
-    # trim the comments first    
-    $footer =~ s/<!--[\s\S]*?--[ \t\n\r]*>//isg;
-    
-    # contain sections that are to be preserved
-    my $preserve_sections = "";   
-    
-    # process the navigation section    
-    my $nav_match_exp = "<div([^>]*)id=(\"|')p-navigation(\"|')(.|\n)*?<\/div>";
-    if (defined $self->{'nav_div_exp'}) {
-      $nav_match_exp = $self->{'nav_div_exp'} if ($self->{'nav_div_exp'} =~ /\S/) ;
-    }
-        
-    if (defined $self->{'delete_nav'} && ($self->{'delete_nav'} eq "1")) {	
-        # do nothing	
-    } else {      
-      if ($footer =~ m/$nav_match_exp/ig) {
-        $preserve_sections = $& ;
-      } else {
-        print $outhandle "Can't find the navigation section with : $nav_match_exp\n";
-      }
-      # if($preserve_sections =~/\S/){
-      #  $preserve_sections .= "</div>";
-      # }            
-    }          
-            
-    # process the searchbox section        
-    my $searchbox_exp = "<div([^>]*)id=(\"|')p-search(\"|')(.|\\n)*?<\/div>";
-    if(defined $self->{'searchbox_div_exp'}) {                
-        $searchbox_exp = $self->{'searchbox_div_exp'} if ($self->{'searchbox_div_exp'} =~ /\S/);
-    }    
-                        
-    my $searchbox_section = "";    
-    $footer =~ m/$searchbox_exp/ig;
-    $searchbox_section = $& if defined $&;    
-    
-    # make the searchbox form work in Greenstone
-    if($searchbox_section =~ /\S/){        
-        # replace action
-        $searchbox_section =~ s/action="([^>]*)"/action="_gwcgi_"/isg;
-                
-        # remove buttons
-        $searchbox_section =~ s/name="search"/name="q"/isg;
-        $searchbox_section =~ s/name="go"//isg;
-        $searchbox_section =~ s/name="fulltext"//isg;
-                
-        # get collection name from $base_dir for c param        
-        $base_dir =~ m/\/collect\/(.+)\//i;
-        my $collection_name = "";
-        $collection_name = $1 if defined $1;
-        
-        # add Greenstone search params
-        my $hidden_params = "<input type=\"hidden\" name=\"a\" value=\"q\"/>\n" 
-            ."<input type=\"hidden\" name=\"c\" value=\"$collection_name\"/>\n";
-            # ."<input type=\"hidden\" name=\"fqf\" value=\"TX\"/>\n"
-            # ."<input type=\"hidden\" name=\"r\" value=\"1\">\n";
-        
-        $searchbox_section =~ s/<form([^>]*)>/<form$1>\n$hidden_params/isg;         
-        
-        # $searchbox_section .= "</div>";
-    } else {
-      print $outhandle "Can't find the searchbox section with : $searchbox_section\n";
-    }        
-    
-    # either delete or replace the searchbox 
-    if(defined $self->{'delete_searchbox'} && $self->{'delete_searchbox'} eq "1") {
-        # do nothing        
-    } else {
-        $preserve_sections .= "\n$searchbox_section\n";
-    } 
-    
-    if($preserve_sections ne ""){
-      $preserve_sections = "<div id=\"column-one\">\n" . $preserve_sections . "\n</div>\n";
-    }
-    $preserve_sections = "</div></div></div>\n" . $preserve_sections . "\n</body>";   
-            
-    $body_text =~ s/$print_footer/$preserve_sections/isg;
-    
-    
-    # delete other forms in the page
-    my @forms;
-    my $form_count = 0;
-    while($body_text =~ m/<form([^>]*)name=("|')([^>"']*)?("|')/isg){
-        next if($3 eq "searchform");
-        $forms[$form_count++] = $&;        
-    }
-    foreach my $form (@forms) {      
-      $body_text =~ s/$form[\s\S]*?<\/form>//m;
-    }
-    
-    # process links. 
-    # because current WGET 1.10 the -k and -E option doesn't work together
-    # need to 'manually' convert the links to relative links
-    # Dealing with 3 types of links:
-    # -- outgoing links
-    #   -- if we have downloaded the target files, link to the internal version (relative link)
-    #   -- otherwise, link to the external version (absolute links)
-    # -- in-page links (relative link)
-    
-    # NOTE: (important)
-    #   must use the MediaWikiDownload in GLI Download Panel to download files from a MediaWiki website
-    #   otherwise, the internal links may have problems
-    
-    # remove the title attribute of <a> tag
-    $body_text =~ s/<a([^>]*)title="(.*?)"/<a$1/isg;
-    
-    # extract all the links
-    my @links;
-    my $link_count = 0;    
-    while($body_text =~ m/(href|src)="([^>\s]*)$url_base\/([^>\s]*)"/ig){        
-        $links[$link_count++] = "$1=\"$2$url_base/$3\"";		
-    }
-    
-    foreach my $cur_link (@links) {     
-        # escape greedy match + character
-        $cur_link =~ s/\+/\\+/isg;
-        
-        $cur_link =~ m/(.+)"([^>]*)$url_base\/([^>\s]*)"/;          
-        my $external_file_path = "$1\"http://$url_base/$3\"";
-           
-        $body_text =~ s/$cur_link/$external_file_path/i; 
-    }
-             
-    # tag links to new wiki pages as red    
-    $body_text =~ s/<a([^>]*)class="new"([^>]*)>/<a$1style="color:red"$2)>/gi;
-    
-    # tag links to pages external of the MediaWiki website as blue
-    $body_text =~ s/<a([^>]*)class='external text'([^>]*)>/<a$1style="color:blue"$2)>/gi;
-        
-    
-    # process the table-of-contents section
-    # if 'show_toc' is set, add Main_Page's toc to the collection's About page, change extra.dm file     
-    # 1. read _content_ macro from about.dm
-    # 2. append the toc, change all links to the Greenstone internal format for relative links 
-    # 3. write to the extra.dm
-    # TODO: we assume the _about:content_ hasn't been specified before
-    #	    so needs to add function to handle when the macro is already in the extra.dm	   
-    if($self->{'show_toc'}==1 && $file =~ m/Main_Page.(html|htm)$/){
-    
-      # extract toc of the Main_Page           	 
-      my $mainpage_toc = "";  
-      my $toc_exp = "<table([^>]*)id=(\"|')toc(\"|')(.|\\n)*</table>\\n";
-      if($self->{'toc_exp'} =~ /\S/){
-         $toc_exp = $self->{'toc_exp'};	     
-      }
-      if($body_text =~ /$toc_exp/){                      	
-        $mainpage_toc = $&;
-      }
-        
-      if($mainpage_toc =~ /\S/) {
-        
-        # change the in-page links to relative links, for example, change <a href="#section1"> to 
-        # <a href="_httpquery_&a=extlink&rl=1&href=http://www.mediawikisite.com/Main_Page.html#section1">           
-        my $file_url_format = $file;
-        $file_url_format =~ s/\\/\//isg; 
-	$file_url_format = "http://" . $file_url_format;
-	   
-        # encode as URL, otherwise doesn't work on Windows
-        $file_url_format =~ s/([^A-Za-z0-9])/sprintf("%%%02X", ord($1))/seg;
-	$mainpage_toc =~ s/<a href="([^>"#]*)#([^>"]*)"/<a href="_httpquery_&a=extlink&rl=1&href=$file_url_format#$2"/isg;
-        
-        
-        # read the collection's extra.dm    
-        my $macro_path = $base_dir;
-        $macro_path =~ s/import$/macros/;    	
-    	my $extradm_file = &util::filename_cat($macro_path, "extra.dm");        
-        
-        my $extra_dm = "";
-        if(open(INPUT, "<$extradm_file")){    	            
-	    while(my $line = <INPUT>){
-		$extra_dm .= $line;
-	    }	    	
-        } else {
-            print $outhandle "can't open file $extradm_file\n";
-        }
-        close(INPUT);
-        
-        # check whether we have changed the macros
-        my @packages = split("package ", $extra_dm);
-        my $about_package = "";
-        foreach my $package (@packages) {
-          $about_package = "package " . $package if($package =~ /^about/);
-        }      
-                
-        my $update_extra_dm = 0;        
-        
-        if( $about_package =~ /\S/ && $about_package =~ m/_content_(\s*){/ && $about_package =~ m/$mainpage_toc/){  
-	   print $outhandle "_content_ macro already changed!!!!\n";
-	}
-        # if extra.dm doesn't have an "about package"
-        elsif ($about_package !~ /\S/) {          
-          # read _content_ macro from $GSDLHOME/macros/about.dm file          
-	  my $global_about_package = $self->read_content_from_about_dm();	    
-            
-          # create the extra _content_ macro for this collection           
-          # add the original content of the _content_ macro
-          $global_about_package =~ m/{(.|\n)*<\/div>\n\n/;
-          
-          # append the new about package to extra.dm
-          $extra_dm .= "\n\npackage about\n_content_$&\n\n";
-          $extra_dm .= "<div class=\"section\">\n$mainpage_toc\n</div>\n</div>\n}";
-          
-          $update_extra_dm = 1;
-        } 
-        # the about package exists, but either doesn't have the _content_ macro or 
-        # the _content_ macro doesn't contain the toc
-        else {        
-          # check if there is a content macro   
-          my $content_macro_existed = 0;
-          $content_macro_existed = ($about_package =~ /(\s*|\n)_content_(\s*){/);
-            
-          # if there is one
-          # append a new section div for toc to the end of the document section                    
-          if($content_macro_existed ==1) {
-            $about_package =~ /(\s*|\n)_content_(\s*){(.|\n)*?}/;
-            my $content_macro = $&;                          
-            my $new_content_macro = $content_macro;
-            $new_content_macro =~ s/<div[^>]*class="document">(.|\n)*<\/div>/<div$1class="document">$2\n\n<div class="section">\n$mainpage_toc\n<\/div>\n<\/div>/;              
-            $extra_dm =~ s/$content_macro/$new_content_macro/mg;                                    
-          }
-          # otherwise, append _content_ macro to the about package
-          else {
-            my $new_about_package = $about_package;            
-            my $content_macro = &read_content_from_about_dm();
-            $content_macro =~ m/{(.|\n)*<\/div>\n\n/;            
-            
-            $new_about_package .= "\n\n_content_$&\n\n";
-            $new_about_package .= "<div class=\"section\">\n$mainpage_toc\n</div>\n</div>\n}";              
-            $extra_dm =~ s/$about_package/$new_about_package/mg;   
-          } 
-          
-          # either the case, we need to update the extra.dm         
-          $update_extra_dm = 1;
-         }          
-                  
-         if($update_extra_dm==1){
-            # write to the extra.dm file of the collection
-            if (open(OUTPUT, ">$extradm_file")) {
-                print OUTPUT $extra_dm;
-            } else {
-                print "can't open $extradm_file\n";
-            }
-            close(OUTPUT);
-         }
-      } else {
-        print $outhandle "Main_Page doesn't have a table-of-contents section\n";
-      }
-    }
-	
-    # If delete_toc is set, remove toc and tof contents.    
-    if (defined $self->{'delete_toc'} && ($self->{'delete_toc'} == 1)){
-	if (defined $self->{'toc_exp'} && $self->{'toc_exp'} =~ /\S/){		
-          # print "\nit matches toc_exp !!\n" if $body_text =~ /$self->{'toc_exp'}/;
-          if ($body_text =~ /$self->{'toc_exp'}/) {
-	    $body_text =~ s/$self->{'toc_exp'}//i;	    
-          }
-	}
-    }        
-    
-    $$textref = "<body" . $body_text;
-    
-    # Wrap the whole page with <div id="wikispecificstyle"></div>
-    # keep the style of this website and don't mess up with the Greenstone styles
-    $$textref =~ s/<body([^>]*)>/$&\n<div id="wikispecificstyle">\n/is;
-    $$textref =~ s/<\/body>/<\/div><\/body>/is;     
-	       
-    $self->SUPER::process(@_);
-    
-    return 1;
-}
-
-
-sub extract_metadata 
-{
-    my $self = shift (@_);
-    my ($textref, $metadata, $doc_obj) = @_;
-    my $outhandle = $self->{'outhandle'};
-    
-    return if (!defined $textref);
-
-    # metadata fields to extract/save. 'key' is the (lowercase) name of the
-    # html meta, 'value' is the metadata name for greenstone to use
-    my %find_fields = ();
-    my ($tag,$value);
-
-    my $orig_field = "";
-    foreach my $field (split /,/, $self->{'metadata_fields'}) {
-	# support tag<tagname>
-	if ($field =~ /^(.*?)<(.*?)>$/) {
-	    # "$2" is the user's preferred gs metadata name
-	    $find_fields{lc($1)}=$2; # lc = lowercase
-	    $orig_field = $1;
-	} else { # no <tagname> for mapping
-	    # "$field" is the user's preferred gs metadata name
-	    $find_fields{lc($field)}=$field; # lc = lowercase
-	    $orig_field = $field;
-	}
-
-	if ($textref =~ m/<o:$orig_field>(.*)<\/o:$orig_field>/i){
-	    $tag = $orig_field;
-	    $value = $1;
-	    if (!defined $value || !defined $tag){
-		#print $outhandle "MediaWikiPlugin: can't find VALUE in \"$tag\"\n";
-		next;
-	    } else {
-		# clean up and add
-		chomp($value); # remove trailing \n, if any
-		$tag = $find_fields{lc($tag)};
-		#print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
-		#    if ($self->{'verbosity'} > 2);
-		$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $value);
-	    }
-	}
-    }
-}
-
-sub safe_escape_regexp
-{
-  my $regexp = shift (@_);
-  
-  # if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-    $regexp =~ s/\\/\\\\/isg;    
-  #} else {
-    $regexp =~ s/\//\\\//isg;         
-  #}
-  return $regexp;
-}
-
-sub read_content_from_about_dm
-{
-    my $self = shift(@_);
-
-  my $about_macro_file = &util::filename_cat($ENV{'GSDLHOME'}, "macros", "about.dm");
-  my $about_page_content = "";
-  if (open(INPUT, "<$about_macro_file")){
-    while (my $line=<INPUT>){
-      $about_page_content .= $line;
-    }
-  } else {
-      my $outhandle = $self->{'outhandle'};
-    print $outhandle "can't open file $about_macro_file\n";
-  }    		
-  close(INPUT);
-            
-  # extract the _content_ macro
-  $about_page_content =~ m/_content_ {(.|\n)*<\/div>\n\n<\/div>\n}/i;
-  $about_page_content = $&;
-  
-  return $about_page_content;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/PostScriptPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/PostScriptPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,260 +1,0 @@
-###########################################################################
-#
-# PostScriptPlugin.pm -- plugin to process PostScript files
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package PostScriptPlugin;
-
-use ConvertBinaryFile;
-use ReadTextFile; # for read_file in convert_post_process. do we need it?
-use sorttools;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @PostScriptPlugin::ISA = ('ConvertBinaryFile', 'ReadTextFile');
-}
-
-my $convert_to_list =
-    [ {	'name' => "auto",
-	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-      {	'name' => "text",
-	'desc' => "{ConvertBinaryFile.convert_to.text}" },
-      { 'name' => "pagedimg_jpg",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_jpg}" },
-      { 'name' => "pagedimg_gif",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_gif}" },
-      { 'name' => "pagedimg_png",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_png}" }
-      ];
-
-my $arguments =
-    [ { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $convert_to_list, 
-	'deft' => "text" },
-      { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => 'regexp',
-	'deft' => &get_default_block_exp() },
-      { 'name' => "extract_date",
-	'desc' => "{PostScriptPlugin.extract_date}",
-	'type' => "flag" },
-      { 'name' => "extract_pages",
-	'desc' => "{PostScriptPlugin.extract_pages}",
-	'type' => "flag" },
-      { 'name' => "extract_title",
-	'desc' => "{PostScriptPlugin.extract_title}",
-	'type' => "flag" } ];
-
-my $options = { 'name'     => "PostScriptPlugin",
-		'desc'     => "{PostScriptPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in postscript format can be replaced with GS-generated html
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@$inputargs,"-title_sub");
-    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    $self->{'file_type'} = "PS";
-
-    if ($self->{'convert_to'} eq "auto") {
-	$self->{'convert_to'} = "text";
-    }
-
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    # following title_sub removes "Page 1" added by ps2ascii, and a leading
-    # "1", which is often the page number at the top of the page. Bad Luck
-    # if your document title actually starts with "1 " - is there a better way?
-    push(@$specific_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
-    push(@$specific_options, "-file_rename_method", "none");
-    
-    if ($secondary_plugin_name eq "TextPlugin") {
-	push(@$specific_options, "-input_encoding", "utf8");
-	push(@$specific_options,"-extract_language") if $self->{'extract_language'};
-    } elsif ($secondary_plugin_name eq "PagedImagePlugin") {
-	push(@$specific_options, "-processing_tmp_files");
-    }
-
-    $self = bless $self, $class;
-    # used for convert_post_process
-    $self->{'input_encoding'} = "auto";
-    $self->{'default_encoding'} = "utf8";
-
-    $self->load_secondary_plugins($class,$secondary_plugin_options, $hashArgOptLists);
-
-    return $self;
-}
-
-
-sub get_default_block_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.(eps)$^;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.ps$^;
-}
-
-# this has been commented out in other plugins. do we need it here?
-# ps files are converted to images (item file should be in utf8) or text (uses pstoascii), so we shouldn't need to ensure utf8
-sub convert_post_process
-{
-    my $self = shift (@_);
-    my ($conv_filename) = @_;
-    
-    my $outhandle=$self->{'outhandle'};
-    
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
-    
-    # read in file ($text will be in utf8)
-    my $text = "";
-    $self->read_file ($conv_filename, $encoding, $language, \$text);
-    
-    # turn any high bytes that aren't valid utf-8 into utf-8.
-    unicode::ensure_utf8(\$text);
-    
-    # Write it out again!
-    $self->utf8_write_file (\$text, $conv_filename);
-}
-
-sub extract_metadata_from_postscript {
-    my $self = shift (@_);
-
-    my ($filename,$doc) = @_;
-
-    my $section = $doc->get_top_section();
-
-    my $title_found = 0;
-    my $pages_found = 0;
-    my $date_found = 0;
-
-    print STDERR "PostScriptPlugin: extracting PostScript metadata from \"$filename\"\n" 
-	if $self->{'verbosity'} > 1;
-
-    open(INPUT, "<$filename");
-    my $date;
-
-    while(my $line =<INPUT>) {
-	if ($self->{'extract_title'} && !$title_found) {
-	    foreach my $word ($line =~ m|Title: ([-A-Za-z0-9@/\/\(\):,. ]*)|g) {
-		my $new_word = $word; 
-		$new_word =~ s/\(Untitled\)//i;
-		$new_word =~ s/\(Microsoft Word\)//i;
-		$new_word =~ s/Microsoft Word//i;
-		$new_word =~ s/^\(//i;
-		$new_word =~ s/\)$//i;
-		$new_word =~ s/^ - //i;
-		if ($new_word ne "") {
-		    $doc->add_utf8_metadata($section, "Title", $new_word );
-		    $title_found = 1;
-		}
-	    }
-	}
-	if ($self->{'extract_date'} && !$date_found) {
-            foreach my $word ($line =~ m/(Creation[-A-Za-z0-9@\/\(\):,. ]*)/g) {
-                if ($word =~ m/ ([A-Za-z][A-Za-z][A-Za-z]) ([0-9 ][0-9])  ?[0-9: ]+ ([0-9]{4})/) {
-                    $date = &sorttools::format_date($2,$1,$3);
-		    if (defined $date) {
-			$doc->add_utf8_metadata($section, "Date", $date );
-		    }
-                }
-                if ($word =~ m/D:([0-9]{4})([0-9]{2})([0-9]{2})[0-9]{6}\)/) {
-                    $date = &sorttools::format_date($3,$2,$1);
-		    if (defined $date) {
-			$doc->add_utf8_metadata($section, "Date", $date );
-		    }
-                }
-                if ($word =~ m/CreationDate: ([0-9]{4}) ([A-Za-z][A-Za-z][A-Za-z]) ([0-9 ][0-9]) [0-9:]*/) {
-                    $date = &sorttools::format_date($3,$2,$1);
-		    if (defined $date) {
-			$doc->add_utf8_metadata($section, "Date", $date );
-		    }
-                }
-		$date_found = 1;
-            }
-	}
-	if ($self->{'extract_pages'} && !$pages_found) {
-	    foreach my $word ($line =~ m/(Pages: [0-9]*)/g) {
-                my $digits = $word;
-                $digits =~ s/[^0-9]//g;
-		if ($digits ne "" && $digits ne "0") {
-		    $doc->add_utf8_metadata($section, "Pages", $digits );
-		    $pages_found = 1;
-		}
-            }
-	}
-    }
-}
-
-# do plugin specific processing of doc_obj 
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my $filename = &util::filename_cat($base_dir,$file);
-    $self->extract_metadata_from_postscript($filename, $doc_obj);
-
-    return $self->SUPER::process(@_);
-
-}
-
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ReadXMLFile.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ReadXMLFile.pm	(revision 26941)
+++ 	(revision )
@@ -1,430 +1,0 @@
-###########################################################################
-#
-# ReadXMLFile.pm -- base class for XML plugins
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ReadXMLFile;
-
-use BasePlugin;
-use doc;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @ReadXMLFile::ISA = ('BasePlugin');
-
-    # ensure Greenstone's CPAN is on the path (but check first) [jmt12]
-    my $inc_paths = join(':', @INC);
-    my $gsdl_perllib_path = $ENV{'GSDLHOME'} . '/perllib/cpan';
-    if ($inc_paths !~ /$gsdl_perllib_path/)
-    {
-      unshift (@INC, $gsdl_perllib_path);
-    }
-}
-
-use XMLParser;
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "xslt",
-	'desc' => "{ReadXMLFile.xslt}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "ReadXMLFile",
-		'desc'     => "{ReadXMLFile.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about creating the XML parser as all we want is the 
-	# list of plugin options
-	return bless $self, $class;
-    }
-
-    my $parser = new XML::Parser('Style' => 'Stream',
-                                 'Pkg' => 'ReadXMLFile',
-                                 'PluginObj' => $self,
-				 'Handlers' => {'Char' => \&Char,
-						'XMLDecl' => \&XMLDecl,
-						'Entity'  => \&Entity,
-						'Doctype' => \&Doctype,
-						'Default' => \&Default
-                                 });
-
-    $self->{'parser'} = $parser;
-
-    return bless $self, $class;
-}
-
-# the inheriting class must implement this method to tell whether to parse this doc type
-sub get_doctype {
-    my $self = shift(@_);
-    die "$self The inheriting class must implement get_doctype method";
-}
-
-
-sub apply_xslt
-{
-    my $self = shift @_;
-    my ($xslt,$filename) = @_;
-    
-    my $outhandle = $self->{'outhandle'};
-
-    my $xslt_filename = $xslt;
-
-    if (! -e $xslt_filename) {
-	# Look in main site directory
-	my $gsdlhome = $ENV{'GSDLHOME'};
-	$xslt_filename = &util::filename_cat($gsdlhome,$xslt);
-    }
-
-    if (! -e $xslt_filename) {
-	# Look in collection directory
-	my $coldir = $ENV{'GSDLCOLLECTDIR'};
-	$xslt_filename = &util::filename_cat($coldir,$xslt);
-    }
-
-    if (! -e $xslt_filename) {
-	print $outhandle "Warning: Unable to find XSLT $xslt\n";
-	if (open(XMLIN,"<$filename")) {
-
-	    my $untransformed_xml = "";
-	    while (defined (my $line = <XMLIN>)) {
-
-		$untransformed_xml .= $line;
-	    }
-	    close(XMLIN);
-	    
-	    return $untransformed_xml;
-	}
-	else {
-	    print $outhandle "Error: Unable to open file $filename\n";
-	    print $outhandle "       $!\n";
-	    return "";
-	}
-	
-    }
-
-    my $bin_java = &util::filename_cat($ENV{'GSDLHOME'},"bin","java");
-    my $jar_filename = &util::filename_cat($bin_java,"xalan.jar");
-    my $xslt_base_cmd = "java -jar $jar_filename";
-    my $xslt_cmd = "$xslt_base_cmd -IN \"$filename\" -XSL \"$xslt_filename\"";
-
-    my $transformed_xml = "";
-
-    if (open(XSLT_IN,"$xslt_cmd |")) {
-	while (defined (my $line = <XSLT_IN>)) {
-
-	    $transformed_xml .= $line;
-	}
-	close(XSLT_IN);
-    }
-    else {
-	print $outhandle "Error: Unable to run command $xslt_cmd\n";
-	print $outhandle "       $!\n";
-    }
-
-    return $transformed_xml;
-
-}
-
-sub can_process_this_file {
-    my $self = shift(@_);
-    my ($filename) = @_;
-
-    if (-f $filename 
-	&& $self->SUPER::can_process_this_file($filename)
-	&& $self->check_doctype($filename)) {
-	return 1; # its a file for us
-    }
-    return 0;
-}
-
-sub check_doctype {
-    my $self = shift (@_);
-    
-    my ($filename) = @_;
-
-    if (open(XMLIN,"<$filename")) {
-	my $doctype = $self->get_doctype();
-	## check whether the doctype has the same name as the root element tag
-	while (defined (my $line = <XMLIN>)) {
-	    ## find the root element
-	    if ($line =~ /<([\w\d:]+)[\s>]/){
-		my $root = $1;
-		if ($root !~ $doctype){
-		    close(XMLIN);
-		    return 0;
-		}
-		else {
-		    close(XMLIN); 
-		    return 1;
-		}
-	    }
-	}
-	close(XMLIN);
-    }
-    
-    return undef; # haven't found a valid line
-    
-}
-
-sub read {
-    my $self = shift (@_);  
-  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-    
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-    $self->{'base_dir'} = $base_dir;
-    $self->{'file'} = $file;
-    $self->{'filename'} = $filename_full_path;
-    $self->{'filename_no_path'} = $filename_no_path;
-    $self->{'processor'} = $processor;
-
-    # this contains metadata passed in from running metadata_read with other plugins (eg from MetadataXMLPlugin)
-    # we are also using it to store up any metadata found during parsing the XML, so that it can be added to the doc obj.
-    $self->{'metadata'} = $metadata;
-
-    if ($self->parse_file($filename_full_path)) {
-	return 1; # processed the file
-    }
-    return -1;
-}
-
-
-sub parse_file {
-    my $self = shift (@_);
-    my ($filename_full_path, $file, $gli) = @_;
-    eval {
-	my $xslt = $self->{'xslt'};
-	if (defined $xslt && ($xslt ne "")) {
-	    # perform xslt
-	    my $transformed_xml = $self->apply_xslt($xslt,$filename_full_path);
-
-	    # feed transformed file (now in memory as string) into XML parser
-	    $self->{'parser'}->parse($transformed_xml);
-	}
-	else {
-	    $self->{'parser'}->parsefile($filename_full_path);
-	}
-    };
-  
-    if ($@) {
-
-	# parsefile may either croak somewhere in XML::Parser (e.g. because
-	# the document is not well formed) or die somewhere in ReadXMLFile or a
-	# derived plugin (e.g. because we're attempting to process a
-	# document whose DOCTYPE is not meant for this plugin). For the
-	# first case we'll print a warning and continue, for the second
-	# we'll just continue quietly
-
-	print STDERR "**** Error is: $@\n";
-
-	my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
-	if (defined $msg) {	
-	    my $outhandle = $self->{'outhandle'};
-	    my $plugin_name = ref ($self);
-	    print $outhandle "$plugin_name failed to process $file ($msg)\n";
-	}
-
-	# reset ourself for the next document
-	$self->{'section_level'}=0;
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1; # error during processing
-    }
-    return 1; # parsing was successful
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.xml$^;
-}
-
-sub StartDocument {$_[0]->{'PluginObj'}->xml_start_document(@_);}
-sub XMLDecl {$_[0]->{'PluginObj'}->xml_xmldecl(@_);}
-sub Entity {$_[0]->{'PluginObj'}->xml_entity(@_);}
-sub Doctype {$_[0]->{'PluginObj'}->xml_doctype(@_);}
-sub StartTag {$_[0]->{'PluginObj'}->xml_start_tag(@_);}
-sub EndTag {$_[0]->{'PluginObj'}->xml_end_tag(@_);}
-sub Text {$_[0]->{'PluginObj'}->xml_text(@_);}
-sub PI {$_[0]->{'PluginObj'}->xml_pi(@_);}
-sub EndDocument {$_[0]->{'PluginObj'}->xml_end_document(@_);}
-sub Default {$_[0]->{'PluginObj'}->xml_default(@_);}
-
-# This Char function overrides the one in XML::Parser::Stream to overcome a
-# problem where $expat->{Text} is treated as the return value, slowing
-# things down significantly in some cases.
-sub Char {
-    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
-    $_[0]->{'Text'} .= $_[1];
-    return undef;
-}
-
-
-# Called at the beginning of the XML document.
-sub xml_start_document {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    $self->open_document();
-}
-
-# Called for XML declarations
-sub xml_xmldecl {
-    my $self = shift(@_);
-    my ($expat, $version, $encoding, $standalone) = @_;
-}
-
-# Called for XML entities
-sub xml_entity {
-  my $self = shift(@_);
-  my ($expat, $name, $val, $sysid, $pubid, $ndata) = @_;
-}
-
-# Called for DOCTYPE declarations - use die to bail out if this doctype
-# is not meant for this plugin
-sub xml_doctype {
-    my $self = shift(@_);
-
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-    die "ReadXMLFile Cannot process XML document with DOCTYPE of $name";
-}
-
-
-# Called for every start tag. The $_ variable will contain a copy of the
-# tag and the %_ variable will contain the element's attributes.
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-}
-
-# Called for every end tag. The $_ variable will contain a copy of the tag.
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-}
-
-# Called just before start or end tags with accumulated non-markup text in
-# the $_ variable.
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-}
-
-# Called for processing instructions. The $_ variable will contain a copy
-# of the pi.
-sub xml_pi {
-    my $self = shift(@_);
-    my ($expat, $target, $data) = @_;
-}
-
-# Called at the end of the XML document.
-sub xml_end_document {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    $self->close_document();
-}
-
-# Called for any characters not handled by the above functions.
-sub xml_default {
-    my $self = shift(@_);
-    my ($expat, $text) = @_;
-}
-
-sub open_document {
-    my $self = shift(@_);
-
-    my $metadata = $self->{'metadata'};
-    my $filename_full_path = $self->{'filename'};
-
-    # create a new document
-    my $doc_obj = $self->{'doc_obj'} = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-
-    my $filename_no_path = $self->{'filename_no_path'};
-    my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($filename_no_path,$metadata,$plugin_filename_encoding);
-
-    $self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-    
-    # do we want other auto metadata here (see BasePlugin.read_into_doc_obj)
-}
-
-sub close_document {
-    my $self = shift(@_);
-    my $doc_obj = $self->{'doc_obj'};
-
-    # do we want other auto stuff here, see BasePlugin.read_into_doc_obj
-
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->extra_metadata ($doc_obj, 
-			   $doc_obj->get_top_section(), 
-			   $self->{'metadata'});
-   
-    # do any automatic metadata extraction
-    $self->auto_extract_metadata ($doc_obj);
-   
-    # add an OID
-    $self->add_OID($doc_obj);
-    
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "XML");
-
-    # process the document
-    $self->{'processor'}->process($doc_obj);
-    
-    $self->{'num_processed'} ++;
-    undef $self->{'doc_obj'};
-    undef $doc_obj; # is this the same as above??
-}
-
-1;
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/TextPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/TextPlugin.pm	(revision 26941)
+++ 	(revision )
@@ -1,235 +1,0 @@
-###########################################################################
-#
-# TextPlugin.pm -- simple text plugin
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# creates simple single-level document. Adds Title metadata 
-# of first line of text (up to 100 characters long).
-
-package TextPlugin;
-
-use ReadTextFile;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-sub BEGIN {
-    @TextPlugin::ISA = ('ReadTextFile');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" } ,
-      { 'name' => "title_sub",
-	'desc' => "{TextPlugin.title_sub}",
-	'type' => "regexp",
-	'deft' => "",
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "TextPlugin",
-		'desc'     => "{TextPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in regular txt format can be replaced with GS-generated html
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.te?xt$^;
-}
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-    
-    my $cursection = $doc_obj->get_top_section();
-    # get title metadata
-    # (don't need to get title if it has been passed
-    # in from another plugin)
-    if (!defined $metadata->{'Title'}) {
-	my $title = $self->get_title_metadata($textref);
-	$doc_obj->add_utf8_metadata ($cursection, "Title", $title);
-    }
-    # Add FileFormat metadata
-    $doc_obj->add_metadata($cursection, "FileFormat", "Text");
-
-    # insert preformat tags and add text to document object    
-    $self->text_to_html($textref); # modifies the text
-    $doc_obj->add_utf8_text($cursection, $$textref);
-
-    return 1;
-}
-
-sub get_title_metadata {
-    my $self = shift (@_);
-    my ($textref) = @_;
-    
-    my ($title) = $$textref;
-    $title =~ /^\s+/s;
-    if (defined $self->{'title_sub'} && $self->{'title_sub'}) {
-	$title =~ s/$self->{'title_sub'}//;
-    }
-    $title =~ /^\s*([^\n]*)/s; $title=$1;
-    if (length($title) > 100) {
-	$title = substr ($title, 0, 100) . "...";
-    }
-    $title =~ s/\[/&\#91;/g;
-    $title =~ s/\[/&\#93;/g;
-    $title =~ s/\</&\#60;/g;
-    $title =~ s/\>/&\#62;/g;
-    
-    return $title;
-}
-
-sub text_to_html {
-    my $self = shift (@_);
-    my ($textref) = @_;
-    
-    # we need to escape the escape character, or else mg will convert into
-    # eg literal newlines, instead of leaving the text as '\n'
-    $$textref =~ s/\\/\\\\/g; # macro language
-    $$textref =~ s/_/\\_/g; # macro language
-    $$textref =~ s/</&lt;/g;
-    $$textref =~ s/>/&gt;/g;
-    
-    # insert preformat tags and add text to document object
-    $$textref = "<pre>\n$$textref\n</pre>";
-}
-
-
-# replace_srcdoc_with_html.pl requires all subroutines that support src_replaceable
-# to contain a method called tmp_area_convert_file - this is indeed the case with all
-# Perl modules that are subclasses of ConvertToPlug.pm, but as we want TextPlugin to also
-# be srcreplaceable and because TextPlugin does not inherit from ConvertToPlug.pm, we have
-# a similar subroutine with the same name here.
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename) = @_;
-    
-    my $outhandle = $self->{'outhandle'};
-    #my $failhandle = $self->{'failhandle'};
-    
-    # derive output filename from input filename
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-
-    # Softlink to collection tmp dir
-    # First find a temporary directory to create the output file in
-    my $tmp_dirname = $dirname;
-    if(defined $ENV{'GSDLCOLLECTDIR'}) {
-	$tmp_dirname = $ENV{'GSDLCOLLECTDIR'};
-    } elsif(defined $ENV{'GSDLHOME'}) {
-	$tmp_dirname = $ENV{'GSDLHOME'};
-    }
-    $tmp_dirname = &util::filename_cat($tmp_dirname, "tmp");
-    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
-
-    # convert to utf-8 otherwise we have problems with the doc.xml file
-    # later on
-    $tailname = $self->SUPER::filepath_to_utf8($tailname) unless &unicode::check_is_utf8($tailname);
-
-    $suffix = lc($suffix);
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix"); 
-    
-    # Make sure we have the absolute path to the input file
-    # (If gsdl is remote, we're given relative path to input file, of the form import/tailname.suffix
-    # But we can't softlink to relative paths. Therefore, we need to ensure that
-    # the input_filename is the absolute path.
-    my $ensure_path_absolute = 1; # true
-
-    # Now make the softlink,  so we don't accidentally damage the input file
-    # softlinking creates a symbolic link to (or, if that's not possible, it makes a copy of) the original
-    &util::soft_link($input_filename, $tmp_filename, $ensure_path_absolute);
-     
-    my $verbosity = $self->{'verbosity'};
-    if ($verbosity > 0) {
-	# need this output statement, as GShell.java's runRemote() sets status to CANCELLED
-	# if there is no output! (Therefore, it only had this adverse affect when running GSDL remotely)
-	print $outhandle "Converting $tailname$suffix to html\n";
-    }
-
-    #my $output_filename = $tailname$output_ext; #output_ext has to be html!
-    my $output_filename = &util::filename_cat($tmp_dirname, $tailname.".html");
-    
-    # Read contents of text file line by line into an array
-    # create an HTML file from the text file
-    # Recreate the original file for writing the updated contents
-    unless(open(TEXT, "<$tmp_filename")) { # open it as a new file for writing
-	print STDERR "TextPlugin.pm: Unable to open and read from $tmp_filename for converting to html...ERROR: $!\n";
-	return ""; # no file name
-    }
-
-    # Read the entire file at once
-    my $text;
-    { 
-	local $/ = undef; # Now can read the entire file at once. From http://perl.plover.com/local.html
-	$text = <TEXT>;   # File is read in as one single 'line'
-    }
-    close(TEXT); # close the file
-
-    # Get the title before embedding the text in pre tags
-    my $title = $self->get_title_metadata(\$text);   
-
-    # Now convert the text 
-    $self->text_to_html(\$text);
-    
-    # try creating this new file writing and try opening it for writing, else exit with error value
-    unless(open(HTML, ">$output_filename")) {  # open the new html file for writing
-	print STDERR "TextPlugin.pm: Unable to create $output_filename for writing $tailname$suffix txt converted to html...ERROR: $!\n";
-	return ""; # no filename
-    }
-    # write the html contents of the text (which is embedded in <pre> tags) out to the file with proper enclosing tags
-    print HTML "<html>\n<head>\n<title>$title</title>\n</head>\n<body>\n";
-    print HTML $text;
-    print HTML "\n</body>\n</html>";
-    close HTML;
-
-    # remove the copy of the original file/remove the symbolic link to original file
-    &util::rm($tmp_filename);
-
-    return $output_filename; # return the output file path
-}
-
-
-1;
