Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/BibTexPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/BibTexPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,832 +1,0 @@
-###########################################################################
-#
-# BibTexPlugin.pm - a plugin for bibliography records in BibTex format
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 2000 Gordon W. Paynter
-# Copyright 1999-2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-# BibTexPlugin reads bibliography files in BibTex format.
-#
-# by Gordon W. Paynter (gwp@cs.waikato.ac.nz), November 2000
-# Based on ReferPlug.  See ReferPlug for geneology.
-#
-# BibTexPlugin creates a document object for every reference a the file.
-# It is a subclass of SplitTextFile, so if there are multiple records, all
-# are read.
-#
-# Modified Dec 2001 by John McPherson:
-#  *  some modifications submitted by Sergey Yevtushenko
-#                      <sergey@intellektik.informatik.tu-darmstadt.de>
-#  *  some non-ascii char support (ie mostly Latin)
-
-
-package BibTexPlugin;
-
-use SplitTextFile;
-use MetadataRead;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-# BibTexPlugin is a sub-class of SplitTextFile.
-sub BEGIN {
-    @BibTexPlugin::ISA = ('MetadataRead', 'SplitTextFile');
-}
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "split_exp",
-	'desc' => "{SplitTextFile.split_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_split_exp(),
-	'reqd' => "no" }
-      ];
-
-my $options = { 'name'     => "BibTexPlugin",
-		'desc'     => "{BibTexPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-# This plugin processes files with the suffix ".bib"
-sub get_default_process_exp {
-    return q^(?i)\.bib$^;
-}
-
-# This plugin splits the input text at blank lines
-sub get_default_split_exp {
-    return q^\n+(?=@)^;
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-
-# The process function reads a single bibliographic record and stores
-# it as a new document.
-
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $cursection = $doc_obj->get_top_section();
-    $self->{'key'} = "default";
-
-    # Check that we're dealing with a valid BibTex record
-    return undef unless ($$textref =~ /^@\w+\{.*\}/s);
-
-    # Ignore things we can't use
-    return 0 if ($$textref =~ /^\@String/);
-
-    # This hash translates BibTex field names into metadata names.  The
-    # BibTex names are taken from the "Local Guide to Latex" Graeme
-    # McKinstry.  Metadata names are consistent with ReferPlug.
-
-    # The author metadata will be stored as one "Creator" entry, but will
-    # also be split up into several individual "Author" fields.
-
-    my %field = (
-		 'address', 'PublisherAddress',
-		 'author', 'Creator',
-		  
-		 'booktitle', 'Booktitle',
-		 'chapter', 'Chapter',
-		 'edition', 'Edition',
-		 'editor', 'Editor', 
-		 'institution', 'Publisher',
-		 'journal', 'Journal',
-		 'month', 'Month',
-		 'number', 'Number',
-		 'pages', 'Pages',
-		 'publisher', 'Publisher',
-		 'school', 'Publisher',
-		 'title', 'Title',
-		 'volume', 'Volume',
-		 'year', 'Year', # Can't use "Date" as this implies DDDDMMYY!
-
-		 'keywords', 'Keywords',
-		 'abstract', 'Abstract',
-		 'copyright', 'Copyright',
-		 'note', 'Note',
-		 'url', 'URL',
-		 );
-
-    # Metadata fields 
-    my %metadata;
-    my ($EntryType, $Creator, $Keywords, $text);
-
-    my $verbosity = $self->{'verbosity'};
-    $verbosity = 0 unless $verbosity;
-
-    # Make sure the text has exactly one entry per line -
-    # append line to previous if it doesn't start with "  <key> = "
-
-    my @input_lines=split(/\r?\n/, $$textref);
-    my @all_lines;
-    my $entry_line=shift @input_lines;
-    foreach my $input_line (@input_lines) {
-	if ($input_line =~ m/^\s*\w+\s*=\s*/) {
-	    # this is a new key
-	    push(@all_lines, $entry_line);
-	    $entry_line=$input_line;
-	} else {
-	    # this is a continuation of previous line
-	    $entry_line .= " " . $input_line;
-	}
-	
-    }
-    # add final line, removing trailing '}'
-    $entry_line =~ s/\}\s*$//;
-    push(@all_lines, $entry_line);
-    push(@all_lines, "}");
-
-    # Read and process each line in the bib file.
-    my ($entryname, $name, $value, $line);
-    foreach $line (@all_lines) {
-
-	# Add each line.  Most lines consist of a field identifer and
-	# then data, and we simply store them, though we treat some
-	# of the fields a bit differently.
-
-	$line =~ s/\s+/ /g;
-	$text .= "$line\n";
-
-
-	print "Processing line = $line \n" if $verbosity>=4;
-
-	# The first line is special, it contains the reference type and OID
-	if ($line =~ /\@(\w+)\W*\{\W*([\*\.\w\d:-]*)\W*$/) {
-	    $EntryType = $1;
-	    my $EntryID = (defined $2) ? $2 : "default";
-	    print "** $EntryType - \"$EntryID\" \n"
-		if ($verbosity >= 4);
-
-	    next;
-	}
-	if ($line =~ /\@/) {
-	    print $outhandle "bibtexplug: suspect line in bibtex file: $line\n"
-		if ($verbosity >= 2);
-	    print $outhandle "bibtexplug: if that's the start of a new bibtex record ammend regexp in bibtexplug::process()\n"
-		if ($verbosity >= 2);
-	}
-	
-	# otherwise, parse the metadata out of this line
-	next unless ($line =~ /^\s*(\w+)\s*=\s*(.*)/);
-	$entryname = lc($1);
-	$value = $2;
-	$value =~ s/,?\s*$//; # remove trailing comma and space
-	if ($value =~ /^"/ && $value =~ /"$/) {
-	    # remove surrounding " marks
-	    $value =~ s/^"//; $value =~ s/"$//;
-	} elsif ($value =~ /^\{/ && $value =~ /\}$/) {
-	    # remove surrounding {} marks
-	    $value =~ s/^\{//; $value =~ s/\}$//;
-	}
-	# special case for year - we only want a 4 digit year, not 
-	# "circa 2004" etc
-	if ($entryname=~ /^year$/i) {
-	    if ($value=~ /(\d{4})/) {
-		$value=$1;
-		$metadata{$entryname} .= "$value";
-	    }
-	} 
-	else {
-	    
-	    $value = &process_latex($value);
-	    # Add this line of metadata
-	    $metadata{$entryname} .= "$value";	
-	}
-    }
-
-    # Add the Entry type as metadata
-    $doc_obj->add_utf8_metadata ($cursection, "EntryType", $EntryType);
-    
-    #Add the fileformat as metadata
-    $doc_obj->add_metadata($cursection, "FileFormat", "BibTex");
-
-
-    # Add the various field as metadata
-    foreach my $entryname (keys %metadata) {
-	next unless (defined $field{$entryname});
-	next unless (defined $metadata{$entryname});	
-	
-	$name = $field{$entryname};
-	$value = $metadata{$entryname};
-
-	if ($name =~ /^Month/) {
-	    $value=expand_month($value);
-	}
-
-	# Several special operatons on metadata follow
-	
-	# Add individual keywords.
-	# The full set of keywords will be added, in due course, as "Keywords".
-	# However, we also want to add them as individual "Keyword" metadata elements.
-	if ($entryname eq "keywords") {
-	    my @keywordlist = split(/,/, $value);
-	    foreach my $k (@keywordlist) {
-		$k = lc($k); 
-		$k =~ s/\s*$//; 
-		$k =~ s/^\s*//; 
-		if ($k =~ /\w/) {
-		    $doc_obj->add_utf8_metadata ($cursection, "Keyword", $k);
-		}
-	    } 
-	}
-	
-	# Add individual authors
-	# The author metadata will be stored as one "Creator" entry, but we
-	# also want to split it into several individual "Author" fields in
-	# "Lastname, Firstnames" format so we can browse it.
-	if ($entryname eq "author") { #added also comparison with editor
-	   
-	    # take care of "et al."...
-	    my $etal='';
-	    if ($value =~ s/\s+(and\s+others|et\.?\s+al\.?)\s*$//i) {
-		$etal=' <em>et. al.</em>';
-	    }
-	    # und here for german language...
-	    # don't use brackets in pattern, else the matched bit becomes
-	    # an element in the list!
-	    my @authorlist = split(/\s+and\s+|\s+und\s+/, $value); 
-	    my @formattedlist = ();
-	    foreach $a (@authorlist) {
-		$a =~ s/\s*$//; 
-		$a =~ s/^\s*//; 
-		# Reformat and add author name
-		next if $a=~ /^\s*$/;
-		
-		# names are "First von Last", "von Last, First"
-		# or "von Last, Jr, First". See the "BibTeXing" manual, page 16
-		my $first="";
-		my $vonlast="";
-		my $jr="";
-
-		if ($a =~ /,/) {
-		    my @parts=split(/,\s*/, $a);
-		    $first = pop @parts;
-		    if (scalar(@parts) == 2) {
-			$jr = pop @parts;
-		    }
-		    $vonlast=shift @parts;
-		    if (scalar(@parts) > 0) {
-			print $outhandle "BibTexPlugin: couldn't parse name $a\n";
-			# but we continue anyway...
-		    }
-		} else { # First von Last
-		    my @words = split(/ /, $a);
-		    while (scalar(@words) > 1 && $words[0] !~ /^[a-z]{2..}/) {
-			$first .= " " . shift (@words);
-		    }
-		    $first =~ s/^\s//;
-		    $vonlast = join (' ', @words); # whatever's left...
-		}
-		my $von="";
-		my $last="";
-		if ($vonlast =~ m/^[a-z]/) { # lowercase implies "von"
-		    $vonlast =~ s/^(([a-z]\w+\s+)+)//;
-		    $von = $1;
-		    if (!defined ($von)) {
-			# some non-English names do start with lowercase
-			# eg "Marie desJardins". Also we can get typos...
-			print $outhandle "BibTexPlugin: couldn't parse surname $vonlast\n";
-			$von="";
-			if ($vonlast =~ /^[a-z]+$/) {
-			    # if it's all lowercase, uppercase 1st.
-			    $vonlast =~ s/^(.)/\u$1/;
-
-			}
-		    }
-		    $von =~ s/\s*$//;
-		    $last=$vonlast;
-		} else {
-		    $last=$vonlast;
-		}
-		my $wholename="$first $von $last $jr";
-		$wholename =~ s/\s+/ /g; # squeeze multiple spaces
-		$wholename =~ s/ $//;
-		push (@formattedlist, $wholename);
-		my $fullname = "$last";
-		$fullname .= " $jr" if ($jr);
-		$fullname .= ", $first";
-		$fullname .= " $von" if ($von);
-
-		# Add each name to set of Authors
-		$doc_obj->add_utf8_metadata ($cursection, "Author", $fullname);
-	    }
-
-	    # Only want at most one "and" in the Creator field
-	    if (scalar(@formattedlist) > 2) {
-		my $lastauthor=pop @formattedlist;
-		$value=join(', ', @formattedlist);
-		$value.=" and $lastauthor";
-	    } else { # 1 or 2 authors...
-		$value=join(" and ",@formattedlist);
-	    }
-	    $value.=$etal; # if there was "et al."
-	}
-
-	# Books and Journals are additionally marked for display purposes
-	if ($entryname eq "booktitle") {
-	    $doc_obj->add_utf8_metadata($cursection, "BookConfOnly", 1);
-	} elsif ($entryname eq "journal") {
-	    $doc_obj->add_utf8_metadata($cursection, "JournalsOnly", 1); 
-	}
-
-	# Add the various fields as metadata	
-	$doc_obj->add_utf8_metadata ($cursection, $name, $value); 
-    
-    }
-
-    # for books and journals...
-    if (!exists $metadata{'title'}) {
-	my $name=$field{'title'}; # get Greenstone metadata name
-	my $value;
-	if (exists $metadata{'booktitle'}) {
-	    $value=$metadata{'booktitle'};
-	} elsif (exists $metadata{'journal'}) {
-	    $value=$metadata{'journal'};
-	}
-	if ($value) {
-	    $doc_obj->add_utf8_metadata ($cursection, $name, $value); 
-	}
-    }
-
-    # Add Date (yyyymmdd) metadata
-    if (defined ($metadata{'year'}) ) {
-	my $date=$metadata{'year'};
-	chomp $date;
-	my $month=$metadata{'month'};
-	if (defined($month)) {
-	    # month is currently 3 letter code or a range...
-	    $month = expand_month($month);
-	    # take the first month found... might not find one!
-	    $month =~ m/_textmonth(\d\d)_/;
-	    $month = $1;
-	}
-	if (!defined($month)) {
-	    $month="00";
-	}
-	$date .= "${month}00";
-	$doc_obj->add_utf8_metadata($cursection, "Date", $date); 
-}
-
-#    # Add the text in BibTex format (all fields)
-    if ($text =~ /\w/) {
-
-	$text =~ s@&@&amp;@g;
-	$text =~ s@<@&lt;@g;
-	$text =~ s@>@&gt;@g;
-	$text =~ s@\n@<br/>\n@g;
-	$text =~ s@\\@\\\\@g;
-
-	$doc_obj->add_utf8_text ($cursection, $text);
-    }
-
-    return 1;
-}
-
-
-
-
-# convert email addresses and URLs into links
-sub convert_urls_into_links{
-   my ($text) = @_;
- 
-   $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
-   $text =~ s/(http:\/\/[\w\d\.\-]+[\/\w\d\.\-]*)/<a href=\"$1\">$1<\/a>/g;
-
-   return $text;
-}
-
-# Clean up whitespace and convert \n charaters to <BR> or <P>
-sub clean_up_whitespaces{
-   my ($text) = @_;
-
-   $text =~ s/%%%%%/<BR> <BR>/g;
-   $text =~ s/ +/ /g;
-   $text =~ s/\s*$//; 
-   $text =~ s/^\s*//; 
-   $text =~ s/\n/\n<BR>/g;
-   $text =~ s/<BR>\s*<BR>/<P>/g;
-
-   return $text;
-}
-
-
-sub convert_problem_characters_without_ampersand{
-    my ($text) = @_;
-    $text =~ s/</&lt;/g;
-    $text =~ s/>/&gt;/g;
-    
-    $text =~ s/\'\'/\"/g; #Latex -specific conversion
-    $text =~ s/\`\`/\"/g; #Latex -specific conversion
-
-    $text =~ s/\"/&quot;/g;
-    $text =~ s/\'/&#8217;/g;
-    $text =~ s/\`/&#8216;/g;
-
-#    $text =~ s/\+/ /g;
-#    $text =~ s/\(/ /g;
-#    $text =~ s/\)/ /g;
-
-    $text =~ s/\\/\\\\/g;
-
-#    $text =~ s/\./\\\./g;
-
-    return $text;
-}
-
-# Convert a text string into HTML.
-
-# The HTML is going to be inserted into a GML file, so we have to be
-# careful not to use symbols like ">", which occurs frequently in email
-# messages (and use &gt instead.
-
-# This function also turns URLs and email addresses into links, and
-# replaces carriage returns with <BR> tags (and multiple carriage returns
-# with <P> tags).
-
-sub text_into_html {
-    my ($text) = @_;
-
-    # Convert problem characters into HTML symbols
-    $text =~ s/&/&amp;/g;
-
-    $text = &convert_problem_characters_without_ampersand( $text );
-
-    # convert email addresses and URLs into links
-    $text = &convert_urls_into_links( $text );
-
-    $text = &clean_up_whitespaces( $text );
-
-    return $text;
-}
-
-
-sub expand_month {
-    my $text=shift;
-
-    # bibtex style files expand abbreviations for months.
-    # Entries can contain more than one month (eg ' month = jun # "-" # aug, ')
-    $text =~ s/jan/_textmonth01_/g;
-    $text =~ s/feb/_textmonth02_/g;
-    $text =~ s/mar/_textmonth03_/g;
-    $text =~ s/apr/_textmonth04_/g;
-    $text =~ s/may/_textmonth05_/g;
-    $text =~ s/jun/_textmonth06_/g;
-    $text =~ s/jul/_textmonth07_/g;
-    $text =~ s/aug/_textmonth08_/g;
-    $text =~ s/sep/_textmonth09_/g;
-    $text =~ s/oct/_textmonth10_/g;
-    $text =~ s/nov/_textmonth11_/g;
-    $text =~ s/dec/_textmonth12_/g;
-
-    return $text;
-}
-
-
-# Convert accented characters, remove { }, interprete some commands....
-# Note!! This is not comprehensive! Also assumes Latin -> Unicode!
-sub process_latex {
-    my ($text) = @_;
-      
-    # note - this is really ugly, but it works. There may be a prettier way
-    # of mapping latex accented chars to utf8, but we just brute force it here.
-    # Also, this isn't complete - not every single possible accented letter
-    # is in here yet, but most of the common ones are.
-
-    my %utf8_chars =
-	(
-	 # acutes
-	 '\'a' => chr(0xc3).chr(0xa1),
-	 '\'c' => chr(0xc4).chr(0x87),
-	 '\'e' => chr(0xc3).chr(0xa9),
-	 '\'i' => chr(0xc3).chr(0xad),
-	 '\'l' => chr(0xc3).chr(0xba),
-	 '\'n' => chr(0xc3).chr(0x84),
-	 '\'o' => chr(0xc3).chr(0xb3),
-	 '\'r' => chr(0xc5).chr(0x95),
-	 '\'s' => chr(0xc5).chr(0x9b),
-	 '\'u' => chr(0xc3).chr(0xba),
-	 '\'y' => chr(0xc3).chr(0xbd),
-	 '\'z' => chr(0xc5).chr(0xba),
-	 # graves
-	 '`a' => chr(0xc3).chr(0xa0),
-	 '`A' => chr(0xc3).chr(0x80),
-	 '`e' => chr(0xc3).chr(0xa8),
-	 '`E' => chr(0xc3).chr(0x88),
-	 '`i' => chr(0xc3).chr(0xac),
-	 '`I' => chr(0xc3).chr(0x8c),
-	 '`o' => chr(0xc3).chr(0xb2),
-	 '`O' => chr(0xc3).chr(0x92),
-	 '`u' => chr(0xc3).chr(0xb9),
-	 '`U' => chr(0xc3).chr(0x99),
-	 # circumflex
-	 '^a' => chr(0xc3).chr(0xa2),
-	 '^A' => chr(0xc3).chr(0x82),
-	 '^c' => chr(0xc4).chr(0x89),
-	 '^C' => chr(0xc4).chr(0x88),
-	 '^e' => chr(0xc3).chr(0xaa),
-	 '^E' => chr(0xc3).chr(0x8a),
-	 '^g' => chr(0xc4).chr(0x9d),
-	 '^G' => chr(0xc4).chr(0x9c),
-	 '^h' => chr(0xc4).chr(0xa5),
-	 '^H' => chr(0xc4).chr(0xa4),
-	 '^i' => chr(0xc3).chr(0xae),
-	 '^I' => chr(0xc3).chr(0x8e),
-	 '^j' => chr(0xc4).chr(0xb5),
-	 '^J' => chr(0xc4).chr(0xb4),
-	 '^o' => chr(0xc3).chr(0xb4),
-	 '^O' => chr(0xc3).chr(0x94),
-	 '^s' => chr(0xc5).chr(0x9d),
-	 '^S' => chr(0xc5).chr(0x9c),
-	 '^u' => chr(0xc3).chr(0xa2),
-	 '^U' => chr(0xc3).chr(0xbb),
-	 '^w' => chr(0xc5).chr(0xb5),
-	 '^W' => chr(0xc5).chr(0xb4),
-	 '^y' => chr(0xc5).chr(0xb7),
-	 '^Y' => chr(0xc5).chr(0xb6),
-	 
-	 # diaeresis
-	 '"a' => chr(0xc3).chr(0xa4),
-	 '"A' => chr(0xc3).chr(0x84),
-	 '"e' => chr(0xc3).chr(0xab),
-	 '"E' => chr(0xc3).chr(0x8b),
-	 '"\\\\i' => chr(0xc3).chr(0xaf),
-	 '"\\\\I' => chr(0xc3).chr(0x8f),
-	 '"o' => chr(0xc3).chr(0xb6),
-	 '"O' => chr(0xc3).chr(0x96),
-	 '"u' => chr(0xc3).chr(0xbc),
-	 '"U' => chr(0xc3).chr(0x9c),
-	 '"y' => chr(0xc3).chr(0xbf),
-	 '"Y' => chr(0xc3).chr(0xb8),
-	 # tilde
-	 '~A' => chr(0xc3).chr(0x83),
-	 '~N' => chr(0xc3).chr(0x91),
-	 '~O' => chr(0xc3).chr(0x95),
-	 '~a' => chr(0xc3).chr(0xa3),
-	 '~n' => chr(0xc3).chr(0xb1),
-	 '~o' => chr(0xc3).chr(0xb5),
-	 # caron - handled specially
-	 # double acute
-	 # ring
-	 # dot
-	 '.c' => chr(0xc4).chr(0x8b),
-	 '.C' => chr(0xc4).chr(0x8a),
-	 '.e' => chr(0xc4).chr(0x97),
-	 '.E' => chr(0xc4).chr(0x96),
-	 '.g' => chr(0xc4).chr(0xa1),
-	 '.G' => chr(0xc4).chr(0xa0),
-	 '.I' => chr(0xc4).chr(0xb0),
-	 '.z' => chr(0xc5).chr(0xbc),
-	 '.Z' => chr(0xc5).chr(0xbb),
-	 # macron
-	 '=a' => chr(0xc4).chr(0x81),
-	 '=A' => chr(0xc4).chr(0x80),
-	 '=e' => chr(0xc4).chr(0x93),
-	 '=E' => chr(0xc4).chr(0x92),
-	 '=i' => chr(0xc4).chr(0xab),
-	 '=I' => chr(0xc4).chr(0xaa),
-	 '=o' => chr(0xc4).chr(0x8d),
-	 '=O' => chr(0xc4).chr(0x8c),
-	 '=u' => chr(0xc4).chr(0xab),
-	 '=U' => chr(0xc4).chr(0xaa),
-	 
-	 # stroke - handled specially - see below
-	 
-	 # cedilla - handled specially
-	 );
-    
-# these are one letter latex commands - we make sure they're not a longer
-# command name. eg {\d} is d+stroke, so careful of \d
-    my %special_utf8_chars = 
-	(
-	 # breve
-	 'u g' => chr(0xc4).chr(0x9f),
-	 'u G' => chr(0xc4).chr(0x9e),
-	 'u i' => chr(0xc4).chr(0xad),
-	 'u I' => chr(0xc4).chr(0xac),
-	 'u o' => chr(0xc5).chr(0x8f),
-	 'u O' => chr(0xc5).chr(0x8e),
-	 'u u' => chr(0xc5).chr(0xad),
-	 'u U' => chr(0xc5).chr(0xac),
-	 # caron
-	 'v c' => chr(0xc4).chr(0x8d),
-	 'v C' => chr(0xc4).chr(0x8c),
-	 'v n' => chr(0xc5).chr(0x88),
-	 'v N' => chr(0xc5).chr(0x87),
-	 'v s' => chr(0xc5).chr(0xa1),
-	 'v S' => chr(0xc5).chr(0xa5),
-	 'v z' => chr(0xc5).chr(0xbe),
-	 'v Z' => chr(0xc5).chr(0xbd),
-	 # cedilla
-	 'c c' => chr(0xc3).chr(0xa7),
-	 'c C' => chr(0xc3).chr(0x87),
-	 'c g' => chr(0xc4).chr(0xa3),
-	 'c G' => chr(0xc4).chr(0xa2),
-	 'c k' => chr(0xc4).chr(0xb7),
-	 'c K' => chr(0xc4).chr(0xb6),
-	 'c l' => chr(0xc4).chr(0xbc),
-	 'c L' => chr(0xc4).chr(0xbb),
-	 'c n' => chr(0xc5).chr(0x86),
-	 'c N' => chr(0xc5).chr(0x85),
-	 'c r' => chr(0xc5).chr(0x97),
-	 'c R' => chr(0xc5).chr(0x96),
-	 'c s' => chr(0xc5).chr(0x9f),
-	 'c S' => chr(0xc5).chr(0x9e),
-	 'c t' => chr(0xc5).chr(0xa3),
-	 'c T' => chr(0xc5).chr(0xa2),
-	 # double acute / Hungarian accent
-	 'H O' => chr(0xc5).chr(0x90),
-	 'H o' => chr(0xc5).chr(0x91),
-	 'H U' => chr(0xc5).chr(0xb0),
-	 'H u' => chr(0xc5).chr(0xb1),
-	 
-	 # stroke 
-	 'd' => chr(0xc4).chr(0x91),
-	 'D' => chr(0xc4).chr(0x90),
-	 'h' => chr(0xc4).chr(0xa7),
-#	 'H' => chr(0xc4).chr(0xa6), # !! this normally(!!?) means Hung. umlaut
-	 'i' => chr(0xc4).chr(0xb1), # dotless lowercase i
-	 'l' => chr(0xc5).chr(0x82),
-	 'L' => chr(0xc5).chr(0x81),
-	 'o' => chr(0xc3).chr(0xb8),
-	 'O' => chr(0xc3).chr(0x98),
-	 't' => chr(0xc5).chr(0xa7),
-	 'T' => chr(0xc5).chr(0xa6),
-	 # german ss/szlig/sharp s
-	 'ss' => chr(0xc3).chr(0x9f),
-	 );
-    
-    # convert latex-style accented characters.
-
-    # remove space (if any) between \ and letter to accent (eg {\' a})
-
-    $text =~ s@(\\[`'="^~\.])\s(\w)@$1$2@g; #`
-		   
-    # remove {} around a single character (eg \'{e})
-    $text =~ s@(\\[`'="^~\.]){(\w)}@{$1$2}@g; #`
-
-    # \, is another way of doing cedilla \c
-    $text =~ s@\\,(.)@\\c $1@g;
-
-    # remove {} around a single character for special 1 letter commands -
-    # need to insert a space. Eg \v{s}  ->  {\v s}
-    $text =~ s@(\\[uvcH]){(\w)}@{$1 $2}@g;
-
-    # only do if the text contains a '\' character. 
-    if ($text =~ m|\\|) {
-	# "normal" accents - ie non-alpha latex tag
-	while ($text =~ m@\\([`'="^~\.])([\w])@) { #`
-	    my $tex="$1$2"; my $char="$2";
-	    my $replacement=$utf8_chars{$tex};
-	    if (!defined($replacement)) {
-		print STDERR "BibTexPlugin: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
-		$replacement=$char;
-	    }
-	    $text =~ s/\\\Q$tex/$replacement/g;
-	}
-
-        # where the following letter matters (eg "sm\o rrebr\o d", \ss{})
-        # only do the change if immediately followed by a space, }, {, or \
-	# one letter accents ( + ss)
-        while ($text =~ m@\\([DdhiLlOoTt]|ss)[{}\s\"\\]@) {
-	    my $tex=$1;
-	    my $replacement=$special_utf8_chars{$tex};
-	    if (!defined($replacement)) {
-		print STDERR "BibTexPlugin: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
-		$replacement=$tex;
-	    }
-	    $text =~ s/\\$tex([{}\s\"\\])/$replacement$1/g;
-
-	}
-
-	# one letter latex accent commands that affect following letter
-        while ($text =~ m@\\([uvcH]) ([\w])@) {
-          my $tex="$1 $2"; my $char="$2";
-          my $replacement=$special_utf8_chars{$tex};
-          if (!defined($replacement)) {
-	      print STDERR "BibTexPlugin: Warning: unknown latex accent \"$tex\" in \"$text\"\n";
-	      $replacement=$char;
-	  }
-          $text =~ s/\\$tex/$replacement/g;
-        }
-    }
-
-    # escape html-sensitive characters
-    $text =~ s@&@&amp;@g;
-    $text =~ s@<@&lt;@g;
-    $text =~ s@>@&gt;@g;
-    $text =~ s/''/"/g; # Latex-specific
-    $text =~ s/``/"/g; # Latex-specific
-    # greenstone-specific
-    $text =~ s@\[@&\#91;@g;
-    $text =~ s@\]@&\#93;@g;
-
-    # remove latex commands
-
-    # explicitly recognised commands
-    $text =~ s@\\ldots@&hellip;@g;
-
-    # maths mode
-    $text =~ s@\$(.*?)\$@&process_latex_math($1)@ge;
-
-    # remove all other commands with optional arguments...
-    $text =~ s@\\\w+(\[.*?\])?\s*@@g;
-    # $text =~ s@\\noopsort{[^}]+\}@@g;
-    # $text =~ s@\\\w+{(([^}]*[^\\])*)}@$1@g; # all other commands
-    
-    # remove latex groupings { } (but not \{ or \} )
-    while ($text =~ s/([^\\])[\{\}]/$1/g) {;} # needed for "...}{..."
-    $text =~ s/^\{//; # remove { if first char
-    
-    # latex characters
-    # spaces - nobr space (~), opt break (\-), append ("#" - bibtex only)
-    $text =~ s/([^\\])~+/$1 /g; # non-breaking space  "~"
-    # optional break "\-"
-    if ($text =~ m/[^&]\#/) { # concat macros (bibtex) but not HTML codes
-	# the non-macro bits have quotes around them - we just remove quotes
-	$text =~ s/([^&])[\"\#]/$1/g;
-    }
-    # dashes. Convert (m|n)-dash into single dash for html.
-    $text =~ s@\-\-+@\-@g;
-
-    # quoted { } chars
-    $text =~ s@\\{@{@g;
-    $text =~ s@\\}@}@g;
-
-    # finally to protect against macro language...
-    $text =~ s@\\@\\\\@g;
-
-    return $text;
-}
-
-
-sub process_latex_math {
-    my $text = shift;
-
-    $text =~ s@\\infty@infinity@g;       # or unicode 0x221E...
-    $text =~ s@\^{(.*?)}@<sup>$1</sup>@g; # superscript
-    $text =~ s@\^([^\{])@<sup>$1</sup>@g;
-    $text =~ s@\_{(.*?)}@<sub>$1</sub>@g; # subscript
-    $text =~ s@\_([^\{])@<sub>$1</sub>@g;
-
-    # put all other command names in italics
-    $text =~ s@\\([\w]+)@<i>$1</i>@g;
-
-    return $text;
-}
-
-sub add_OID {
-    my $self = shift (@_);
-    my ($doc_obj, $id, $segment_number) = @_;
-    
-    if ( $self->{'key'} eq "default") {
-	$self->SUPER::add_OID(@_);
-#	$doc_obj->set_OID("$id\_$segment_number");
-    } else {
-	$doc_obj->set_OID($self->{'key'});
-    }
-}
-
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/CONTENTdmPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/CONTENTdmPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,822 +1,0 @@
-###########################################################################
-#
-# CONTENTdmPlugin.pm -- reasonably with-it pdf plugin
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999-2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package CONTENTdmPlugin;
-
-use ConvertBinaryFile;
-use ReadXMLFile;
-
-use unicode;
-use ghtml;
-
-use strict;
-no strict 'refs'; # so we can use a var for filehandles (eg STDERR)
-
-
-use XMLParser;
-
-# inherit ReadXMLFile for the apply_xslt method
-sub BEGIN {
-    @CONTENTdmPlugin::ISA = ('ConvertBinaryFile', 'ReadXMLFile');
-}
-
-
-my $convert_to_list =
-    [ 
-#      {	'name' => "auto",
-#	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-#      {	'name' => "html",
-#	'desc' => "{ConvertBinaryFile.convert_to.html}" },
-#      {	'name' => "text",
-#	'desc' => "{ConvertBinaryFile.convert_to.text}" },
-      { 'name' => "pagedimg",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg}"},
-      ];
-
-
-
-my $arguments = 
-      [
-       { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $convert_to_list, 
-	'deft' => "html" },	 
-      { 'name' => "xslt",
-	'desc' => "{ReadXMLFile.xslt}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-       { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_block_exp() }
-];
-
-my $options = { 'name'     => "CONTENTdmPlugin",
-		'desc'     => "{CONTENTdmPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		# CONTENTdmPlugin is one of the few ConvertBinaryFile subclasses whose source doc can't be replaced by a GS-generated html
-		'srcreplaceable' => "no",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@$inputargs,"-title_sub");
-    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my @arg_array = @$inputargs;
-    my $self = new ConvertBinaryFile($pluginlist,$inputargs,$hashArgOptLists);
-    
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    my $parser = new XML::Parser('Style' => 'Stream',
-				 'Pkg' => 'ReadXMLFile',
-				 'PluginObj' => $self,
-				 'Handlers' => {'Char' => \&ReadXMLFile::Char,
-						'XMLDecl' => \&ReadXMLFile::XMLDecl,
-						'Entity'  => \&ReadXMLFile::Entity,
-						'Doctype' => \&ReadXMLFile::Doctype,
-						'Default' => \&ReadXMLFile::Default
-						});  
-    $self->{'parser'} = $parser;
-
-
-    $self->{'rdf_desc'} = undef;
-    $self->{'about_key'} = undef;
-    $self->{'metadata_name'} = undef;
-    $self->{'metadata_value'} = undef;
-
-    # do we only allow one option??
-    $self->{'convert_to'} = "pagedimg";
-    $self->{'convert_to_plugin'} = "PagedImagePlugin";
-    $self->{'convert_to_ext'} = "jpg";
-    
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    push(@$specific_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
-    push(@$specific_options, "-create_thumbnail", "true", "-create_screenview", "true");
-    push(@$specific_options, "-file_rename_method", "none");
-    push(@$specific_options, "-processing_tmp_files");
-
-#    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-#    if (!defined $secondary_plugin_options->{'PagedImagePlugin'}){
-#	$secondary_plugin_options->{'PagedImagePlugin'} = [];
-#    }
-#    my $pagedimg_options = $secondary_plugin_options->{'PagedImagePlugin'}; 
-#    push(@$pagedimg_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
-#    push(@$pagedimg_options, "-create_thumbnail", "true", "-create_screenview", "true");
-#    push(@$pagedimg_options, "-file_rename_method", "none");
-#    push(@$pagedimg_options, "-processing_tmp_files");
-    $self = bless $self, $class;
-    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
-    return $self;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.rdf$^;
-}
-
-sub get_default_block_exp {
-    return q^(?i)\.(jpg|jpeg|gif)$^;
-}
-
-
-
-sub rdf_desc_to_id
-{
-    my $self = shift (@_);
-    my ($rdf_desc) = @_;
-
-    my $rdf_id = {};
-
-    # initialise any .cpd (=complex multi page) structures 
-
-    foreach my $about_key (keys %{$rdf_desc}) {
-	if ($about_key =~ m/\.cpd$/) {
-	    my $about = $rdf_desc->{$about_key};
-	    my $id    = $about->{'dc:identifier'};
-
-	    if ($id =~ m/^\s*$/) {
-		# missing id, make one up based on about attribute
-
-		my ($tailname, $dirname, $suffix)
-		    = &File::Basename::fileparse($about_key, "\\.[^\\.]+\$");
-
-		$id = "about:$tailname";
-	    }
-
-	    $rdf_id->{$id} = $about;
-	    $rdf_id->{$id}->{'ex:filename'} = $about_key;
-	    $rdf_id->{$id}->{'ex:type'} = "complex";
-	    $rdf_id->{$id}->{'pages'} = [];
-	}
-
-    }
-
-    # now add in *non* .cpd items
-
-    foreach my $about_key (keys %{$rdf_desc}) {
-	if ($about_key !~ m/\.cpd$/) {
-	    my $about = $rdf_desc->{$about_key};	    
-	    my $id    = $about->{'dc:identifier'};
-
-
-	    if ($id =~ m/^\s*$/) {
-		# missing id, make one up based on about attribute
-
-		my ($tailname, $dirname, $suffix)
-		    = &File::Basename::fileparse($about_key, "\\.[^\\.]+\$");
-
-		$id = "about:$tailname";
-	    }
-
-	    if (defined $rdf_id->{$id}) {
-		$about->{'ex:filename'} = $about_key;
-
-		# dealing with complex multi-page situation
-		# Add to existing structure
-
-		my $pages = $rdf_id->{$id}->{'pages'};
-		push(@$pages,$about)
-	    }
-	    else {
-		# New entry
-
-		$rdf_id->{$id} = $about;
-		$rdf_id->{$id}->{'ex:type'} = "simple";	    
-		$rdf_id->{$id}->{'ex:filename'} = $about_key;
-	    }
-	}
-	
-    }
-
-    return $rdf_id;
-}
-
-
-sub metadata_table_txt_file
-{
-    my $self = shift (@_);
-    my ($output_root,$page_num) = @_;
-
-    my $txt_filename = $output_root."_page_$page_num.txt";
-
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($txt_filename, "\\.[^\\.]+\$");
-
-    my $txt_file = "$tailname$suffix";
-
-    return $txt_file;
-}
-
-
-sub output_metadata_table 
-{
-    my $self = shift (@_);
-    my ($page,$page_num,$tmp_dirname,$txt_file) = @_;
-    
-    my $txt_filename = &util::filename_cat($tmp_dirname,$txt_file);
-
-    open(TOUT,">$txt_filename") 
-	|| die "Error: unable to write metadata data out as txt file $txt_filename: $!\n";
-    
-    print TOUT $page->{'MetadataTable'};
-    delete $page->{'MetadataTable'};
-
-    close (TOUT);
-}
-
-
-sub rdf_id_to_item_file
-{
-    my $self = shift (@_);
-    my ($rdf_id,$tmp_dirname,$output_root) = @_;
-
-    my $item_file_list = [];
-    
-    foreach my $id (keys %{$rdf_id}) {
-
-	my $id_safe = $id;
-	$id_safe =~ s/ /-/g;
-
-	my $output_filename = $output_root."_$id_safe.item";
-	open(FOUT,">$output_filename") 
-	    || die "Unable to open $output_filename: $!\n";
-	
-
-	print FOUT "<PagedDocument>\n";
-
-	my $rdf_doc = $rdf_id->{$id};
-	foreach my $metadata_name (keys %$rdf_doc) {
-
-
-	    next if ($metadata_name eq "pages");
-
-	    my $metadata_value = $rdf_doc->{$metadata_name};
-
-	    # convert ns:name to ns.Name
-	    $metadata_name =~ s/^(.*?):(.*)/$1\.\u$2/; 
-
-	    print FOUT "  <Metadata name=\"$metadata_name\">$metadata_value</Metadata>\n";
-	}
-
-	if ($rdf_doc->{'ex:type'} eq "complex") {
-	    my $pages = $rdf_doc->{'pages'};
-	    my $page_num = 1;
-
-	    foreach my $page (@$pages) {
-
-		my $imgfile = $page->{'ex:filename'};
-		if ($imgfile =~ m/(http|ftp):/) {
-		    $imgfile = "empty.jpg";
-		}
-		else {
-		    $imgfile = &util::filename_cat("..","import",$imgfile);
-		}
-		
-		my $txt_file 
-		    = $self->metadata_table_txt_file($output_root,$page_num);
-
-		$self->output_metadata_table($page,$page_num,
-					     $tmp_dirname,$txt_file);
-
-
-		print FOUT "  <Page pagenum=\"$page_num\" imgfile=\"$imgfile\" txtfile=\"$txt_file\">\n";
-
-		foreach my $metadata_name (keys %$page) {
-		
-		    my $metadata_value = $rdf_doc->{$metadata_name};
-		    # convert ns:name to ns.Name
-		    $metadata_name =~ s/^(.*?):(.*)/$1\.\u$2/; 
-
-		    print FOUT "  <Metadata name=\"$metadata_name\">$metadata_value</Metadata>\n";
-		}
-		
-
-		$page_num++;
-
-
-		print FOUT "  </Page>\n";
-	    }
-	}
-	else {
-	    # simple
-	    # duplicate top-level metadata for now plus image to bind to
-
-	    my $imgfile = $rdf_doc->{'ex:filename'};
-	    if ($imgfile =~ m/(http|ftp):/) {
-		$imgfile = "empty.jpg";
-	    }
-	    else {
-		$imgfile = &util::filename_cat("..","import",$imgfile);
-	    }
-
-
-	    my $txt_file = $self->metadata_table_txt_file($output_root,1);
-	    $self->output_metadata_table($rdf_doc,1,$tmp_dirname,$txt_file);
-
-	    print FOUT "  <Page pagenum=\"1\" imgfile=\"$imgfile\" txtfile=\"$txt_file\">\n";	    
-	    foreach my $metadata_name (keys %$rdf_doc) {
-
-		my $metadata_value = $rdf_doc->{$metadata_name};
-		
-		# convert ns:name to ns.Name
-		$metadata_name =~ s/^(.*?):(.*)/$1\.\u$2/; 
-
-		print FOUT "  <Metadata name=\"$metadata_name\">$metadata_value</Metadata>\n";
-	    }
-	    print FOUT "  </Page>\n";
-
-	}
-
-	print FOUT "</PagedDocument>\n";
-	close(FOUT);
-
-	push(@$item_file_list,$output_filename);
-
-    }
-    
-
-    return $item_file_list;
-}
-
-
-
-sub xml_area_convert_file
-{
-    my $self = shift (@_);
-    my ($input_filename, $tmp_dirname, $output_root) = @_;
-
-    eval {
-	# Build up hash table/tree of all records
-
-	my $xslt = $self->{'xslt'};
-	if (defined $xslt && ($xslt ne "")) {
-	    # perform xslt
-	    my $transformed_xml = $self->apply_xslt($xslt,$input_filename);
-
-	    open(TOUT,">/tmp/tout.xml") 
-		|| die "Unable to open /tmp/tout.xml: $!\n";
-	    print TOUT $transformed_xml;
-	    close(TOUT);
-
-
-	    # feed transformed file (now in memory as string) into XML parser
-	    $self->{'parser'}->parse($transformed_xml);
-	}
-	else {
-	    $self->{'parser'}->parsefile($input_filename);
-	}
-    };
-  
-    if ($@) {
-
-	# parsefile may either croak somewhere in XML::Parser (e.g. because
-	# the document is not well formed) or die somewhere in XMLPlug or a
-	# derived plugin (e.g. because we're attempting to process a
-	# document whose DOCTYPE is not meant for this plugin). For the
-	# first case we'll print a warning and continue, for the second
-	# we'll just continue quietly
-
-	print STDERR "**** Error is: $@\n";
-
-	my $file = $self->{'file'};
-
-	my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
-	if (defined $msg) {	
-	    my $outhandle = $self->{'outhandle'};
-	    my $plugin_name = ref ($self);
-	    print $outhandle "$plugin_name failed to process $file ($msg)\n";
-	}
-
-	my $gli = $self->{'gli'};
-
-	# reset ourself for the next document
-	$self->{'section_level'}=0;
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return ("fail",undef); # error during processing
-    }
-
-    my $rdf_desc = $self->{'rdf_desc'};    
-
-#    foreach my $about_key (keys %{$rdf_desc}) {
-#	my $about = $rdf_desc->{$about_key};
-#	foreach my $metadata_name (keys %{$about}) {
-#
-#	    my $metadata_value = $about->{$metadata_name};
-##	    print STDERR " $metadata_name: $metadata_value\n";
-#	}
-#    }
-
-
-    # Merge entries with same name
-
-
-    my $merged_rdf_id = $self->rdf_desc_to_id($rdf_desc);
-
-#    foreach my $about_key (keys %{$merged_rdf_id}) {
-#	my $about = $merged_rdf_id->{$about_key};
-#	foreach my $metadata_name (keys %{$about}) {
-#
-#	    my $metadata_value = $about->{$metadata_name};
-##	    print STDERR " $metadata_name: $metadata_value\n";
-#	}
-#    }
-
-
-
-    my $item_files = $self->rdf_id_to_item_file($merged_rdf_id,$tmp_dirname, 
-						$output_root);
-
-    return ("item",$item_files);
-}
-
-
-# Override ConvertBinaryFile tmp_area_convert_file() to provide solution specific 
-# to CONTENTdm
-#
-# A better (i.e. in the future) solution would be to see if this can be
-# shifted into gsConvert.pl so there is no need to override the
-# default tmp_area_convert_file()
-
-
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename, $textref) = @_;
-
-    # is textref ever used?!?
-
-    my $outhandle = $self->{'outhandle'};
-    my $convert_to = $self->{'convert_to'};
-    my $failhandle = $self->{'failhandle'};
-    my $convert_to_ext = $self->{'convert_to_ext'};
-    
-    # softlink to collection tmp dir
-    my $tmp_dirname 
-	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
-    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
-
-    # derive tmp filename from input filename
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-
-    # Remove any white space from filename -- no risk of name collision, and
-    # makes later conversion by utils simpler. Leave spaces in path...
-    # tidy up the filename with space, dot, hyphen between
-    $tailname =~ s/\s+//g; 
-    $tailname =~ s/\.+//g;
-    $tailname =~ s/\-+//g;
-
-    $tailname = $self->SUPER::filepath_to_utf8($tailname) unless &unicode::check_is_utf8($tailname);
-    $suffix = lc($suffix);
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
-
-    &util::soft_link($input_filename, $tmp_filename);
-    my $verbosity = $self->{'verbosity'};
-    if ($verbosity > 0) {
-	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
-    }
-
-    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
-    
-    # call xml_area_convert_file rather than gsConvert.pl
-
-    my $output_root = &util::filename_cat($tmp_dirname, "$tailname");
-
-    my ($output_type,$item_files) 
-	= $self->xml_area_convert_file($tmp_filename,$tmp_dirname,$output_root);
-
-
-    my $fakeimg_filename = &util::filename_cat($dirname, "empty.jpg");
-    my $fakeimg_tmp_filename = &util::filename_cat($tmp_dirname, "empty.jpg");
-
-    print STDERR "***** No source image identified with item\n";
-
-    print STDERR "***** Using default \"no image available\" $fakeimg_filename -> $fakeimg_tmp_filename\n";
-
-    &util::soft_link($fakeimg_filename, $fakeimg_tmp_filename);
-   
-    # continue as before ...
-
-    # remove symbolic link to original file
-    &util::rm($tmp_filename);
-
-    # Check STDERR here
-    chomp $output_type;
-    if ($output_type eq "fail") {
-	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
-	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
-	$self->{'num_not_processed'} ++;
-	if (-s "$errlog") {
-	    open(ERRLOG, "$errlog");
-	    while (<ERRLOG>) {
-		print $outhandle "$_";
-	    }
-	    print $outhandle "\n";
-	    close ERRLOG;
-	}
-	&util::rm("$errlog") if (-e "$errlog");
-	return [];
-    }
-
-    # store the *actual* output type and return the output filename
-    # it's possible we requested conversion to html, but only to text succeeded
-    #$self->{'convert_to_ext'} = $output_type;
-    if ($output_type =~ /html/i) {
-	$self->{'converted_to'} = "HTML";
-    } elsif ($output_type =~ /te?xt/i) {
-	$self->{'converted_to'} = "Text";
-    } elsif ($output_type =~ /item/i){
-	$self->{'converted_to'} = "PagedImage";
-    }
-
-    
-    return $item_files;
-}
-
-
-
-
-# Override ConvertBinaryFile (ie BasePlugin) read
-# Needed so multiple .item files generated are sent down secondary plugin 
-# and the resulting doc_objs all processed.
-
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-
-    $self->{'gli'} = $gli;
-    $self->{'file'} = $file;
-
-    my $successful_rv = -1;
-
-    my $outhandle = $self->{'outhandle'};
-    
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-
-    # read() deviates at this point from ConvertBinaryFile
-    # Need to work with list of filename returned
-
-    my $output_ext = $self->{'convert_to_ext'};
-    my $conv_filename_list = [];
-
-    $conv_filename_list = $self->tmp_area_convert_file($output_ext, $filename_full_path);
-
-    if (scalar(@$conv_filename_list)==0) {
-	return -1;
-    } # had an error, will be passed down pipeline 
-
-    foreach my $conv_filename ( @$conv_filename_list ) {
-	if (! -e "$conv_filename") {return -1;} 
-	$self->{'conv_filename'} = $conv_filename; # is this used anywhere?
-	$self->convert_post_process($conv_filename);
-    
-	my $secondary_plugins =  $self->{'secondary_plugins'};
-	my $num_secondary_plugins = scalar(keys %$secondary_plugins);
-
-	if ($num_secondary_plugins == 0) {
-	    print $outhandle "Warning: No secondary plugin to use in conversion.  Skipping $file\n";
-	    return 0; # effectively block it
-	}
-
-	my @plugin_names = keys %$secondary_plugins;
-	my $plugin_name = shift @plugin_names;
-	
-	if ($num_secondary_plugins > 1) {
-	    print $outhandle "Warning: Multiple secondary plugins not supported yet!  Choosing $plugin_name\n.";
-	}
-	
-	my $secondary_plugin = $secondary_plugins->{$plugin_name};
-
-	# note: metadata is not carried on to the next level
-	my ($rv,$doc_obj) 
-	    = $secondary_plugin->read_into_doc_obj ($pluginfo,"", $conv_filename, 
-						    $block_hash, $metadata, $processor, $maxdocs, $total_count,
-						    $gli);
-
-	print STDERR "**** $conv_filename => returned rv = $rv\n";
-
-	if ((defined $rv) && ($rv>=0)) {
-	    $successful_rv = 1;
-	}
-
-	# Override previous gsdlsourcefilename set by secondary plugin
-	my $collect_file = &util::filename_within_collection($filename_full_path);
-	my $collect_conv_file = &util::filename_within_collection($conv_filename);
-	$doc_obj->set_source_filename ($collect_file, $self->{'file_rename_method'}); 
-	$doc_obj->set_converted_filename($collect_conv_file);
-	
-	my ($filemeta) = $file =~ /([^\\\/]+)$/;
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-	$self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-	$doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-	$doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FileSize", (-s $filename_full_path));
-	
-	if ($self->{'cover_image'}) {
-	    $self->associate_cover_image($doc_obj, $filename_full_path);
-	}
-	
-	# do plugin specific processing of doc_obj
-	unless (defined ($self->process(undef, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
-	    print STDERR "***** process returned undef: $base_dir $file\n";
-	    print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	    return -1;
-	}
-	# do any automatic metadata extraction
-	$self->auto_extract_metadata ($doc_obj);
-
-	# have we found a Title??
-	$self->title_fallback($doc_obj,$doc_obj->get_top_section(),$filemeta);
-
-	# add an OID
-	$self->add_OID($doc_obj);
-	# process the document
-	$processor->process($doc_obj);
-
-	$self->{'num_processed'} ++;
-    }
-
-    return $successful_rv;
-}
-
-sub process {
-
-    return 1;
-}
-
-# do we need this? sec pluginn process would have already been called as part of read_into_doc_obj??
-sub process_old {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    
-    my $secondary_plugins =  $self->{'secondary_plugins'};
-    my @plugin_names = keys %$secondary_plugins;
-    my $plugin_name = shift @plugin_names; # already checked there is only one
-	
-    my $secondary_plugin = $secondary_plugins->{$plugin_name};
-	
-    my $result = $secondary_plugin->process(@_);
-
-    return $result;
-}
-
-
-# Called at the beginning of the XML document.
-sub xml_start_document {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    $self->{'rdf_desc'} = {};
-}
-
-
-# Called for DOCTYPE declarations - use die to bail out if this doctype
-# is not meant for this plugin
-sub xml_doctype {
-    my $self = shift(@_);
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    die "" if ($name !~ /^rdf:RDF$/);
-
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "CONTENTdmPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
-
-}
-
-# Called for every start tag. The $_ variable will contain a copy of the
-# tag and the %_ variable will contain the element's attributes.
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    if ($element eq "rdf:Description") {
-
-	my $about_key = $_{'about'};
-
-	my $rdf_desc = $self->{'rdf_desc'};
-	$rdf_desc->{$about_key} = {};
-
-	$self->{'about_key'}  = $about_key;
-	$self->{'index_text'} = "";
-	$self->{'pp_text'}    = "<table width=\"100%\">\n";
-
-
-    }
-    elsif (defined $self->{'about_key'}) {	
-	$self->{'metadata_name'} = $element;
-	$self->{'metadata_value'} = "";
-    }
-
-}
-
-# Called for every end tag. The $_ variable will contain a copy of the tag.
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    if ($element eq "rdf:Description") {
-	$self->{'pp_text'} .= "</table>\n";
-	## ghtml::htmlsafe($self->{'pp_text'});
-
-
-	my $about_key = $self->{'about_key'};
-	my $about = $self->{'rdf_desc'}->{$about_key};
-	$about->{'IndexText'}     = $self->{'index_text'};
-	$about->{'MetadataTable'} = $self->{'pp_text'};
-
-
-	$self->{'about_key'}   = undef;
-	$self->{'index_text'}  = undef;
-	$self->{'pp_text'}     = undef;
-
-    }
-    elsif (defined $self->{'metadata_name'}) {
-	my $metadata_name = $self->{'metadata_name'};
-	if ($element eq $metadata_name) {
-	    my $metadata_value = $self->{'metadata_value'};
-
-	    my $about_key = $self->{'about_key'};
-	    my $about = $self->{'rdf_desc'}->{$about_key};
-	    $about->{$metadata_name} = $metadata_value;
-
-	    $self->{'index_text'} .= "$metadata_value\n";
-	    $self->{'pp_text'} .= "  <tr><td>$metadata_name</td><td>$metadata_value</td></tr>\n";
-
-	    $self->{'metadata_name'}  = undef;
-	    $self->{'metadata_value'} = undef;
-	}
-    }
-}
-
-# Called just before start or end tags with accumulated non-markup text in
-# the $_ variable.
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    if (defined $self->{'metadata_name'}) {
-	$self->{'metadata_value'} .= $_;
-    }
-}
-
-# Called at the end of the XML document.
-sub xml_end_document {
-    my $self = shift(@_);
-    my ($expat) = @_;
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ConvertBinaryFile.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ConvertBinaryFile.pm	(revision 26940)
+++ 	(revision )
@@ -1,504 +1,0 @@
-###########################################################################
-#
-# ConvertBinaryFile.pm -- plugin that facilitates conversion of binary files
-# through gsConvert.pl 
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# This plugin is inherited by such plugins as WordPlugin, PowerPointPlugin, 
-# PostScriptPlugin, 
-# RTFPlugin and PDFPlugin. It facilitates the conversion of these document types 
-# to either HTML, Text or a series of images. It works by dynamically loading 
-# an appropriate secondary plugin (HTMLPlug, StructuredHTMLPlug, 
-# PagedImagePlugin or TextPlugin) based on the plugin argument 'convert_to'. 
-
-package ConvertBinaryFile;
-
-use AutoExtractMetadata;
-use ghtml;
-use HTMLPlugin;
-use TextPlugin;
-use PagedImagePlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-use util;
-
-
-sub BEGIN {
-    @ConvertBinaryFile::ISA = ('AutoExtractMetadata');
-}
-
-my $convert_to_list =
-    [ {	'name' => "auto",
-	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-      {	'name' => "html",
-	'desc' => "{ConvertBinaryFile.convert_to.html}" },
-      {	'name' => "text",
-	'desc' => "{ConvertBinaryFile.convert_to.text}" }
-      ];
-
-my $arguments =
-    [ { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $convert_to_list, 
-	'deft' => "auto" },
-      { 'name' => "keep_original_filename",
-	'desc' => "{ConvertBinaryFile.keep_original_filename}",
-	'type' => "flag" },
-      { 'name' => "title_sub",
-	'desc' => "{HTMLPlugin.title_sub}",
-	'type' => "string", 
-	#'type' => "regexp",
-	'deft' => "" },
-      { 'name' => "apply_fribidi",
-	'desc' => "{ConvertBinaryFile.apply_fribidi}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "use_strings",
-	'desc' => "{ConvertBinaryFile.use_strings}",
-	'type' => "flag",
-	'reqd' => "no" },
-      ];
-
-my $options = { 'name'     => "ConvertBinaryFile",
-		'desc'     => "{ConvertBinaryFile.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub load_secondary_plugins
-{
-    my $self = shift (@_);
-    my ($class,$input_args,$hashArgOptLists) = @_;
-
-    my @convert_to_list = split(",",$self->{'convert_to_plugin'});
-    my $secondary_plugins = {};
-    # find the plugin
-
-    foreach my $convert_to (@convert_to_list) {
-	# load in "convert_to" plugin package
-	my $plugin_class = $convert_to;
-	my $plugin_package = $plugin_class.".pm";
-
-	my $colplugname = undef;
-	if (defined $ENV{'GSDLCOLLECTDIR'}) {
-	    $colplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},
-					       "perllib","plugins", 
-					       $plugin_package);
-	}
-
-	my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'},
-					       "perllib","plugins", 
-					       $plugin_package);
-
-	if ((defined $colplugname) && (-e $colplugname)) { require $colplugname;}
-	elsif (-e $mainplugname) { require $mainplugname; }
-	else {
-	    &gsprintf(STDERR, "{plugin.could_not_find_plugin}\n",
-		      $plugin_class);
-	    die "\n";
-	}
-
-	# call its constructor with extra options that we've worked out!
-	my $arglist = $input_args->{$plugin_class};
-
-	my ($secondary_plugin);
-	eval("\$secondary_plugin = new $plugin_class([],\$arglist)");
-	die "$@" if $@;
-	$secondary_plugins->{$plugin_class} = $secondary_plugin;
-    }
-    $self->{'secondary_plugins'} = $secondary_plugins;
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    my $classPluginName = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new AutoExtractMetadata($pluginlist, $inputargs, $hashArgOptLists);
-   
-    return bless $self, $class;
-}
-
-# should be called by subclasses after checking and setting 
-# $self->{'convert_to'}
-sub set_standard_convert_settings {
-    my $self =shift (@_);
-    
-    my $convert_to = $self->{'convert_to'};
-    if ($convert_to eq "auto") {
-	$convert_to = "html";
-	$self->{'convert_to'} = "html";
-    }
-
-    if ($convert_to =~ /^html/) { # may be html or html_multi
-	$self->{'convert_to_plugin'} = "HTMLPlugin";
-	$self->{'convert_to_ext'} = "html";
-    } elsif ($convert_to eq "text") {
-	$self->{'convert_to_plugin'} = "TextPlugin";
-	$self->{'convert_to_ext'} = "txt";
-    } elsif ($convert_to eq "structuredhtml") {
-	$self->{'convert_to_plugin'} = "StructuredHTMLPlugin";
-	$self->{'convert_to_ext'} = "html";
-    } elsif ($convert_to =~ /^pagedimg/) {
-	$self->{'convert_to_plugin'} = "PagedImagePlugin";
-	my ($convert_to_ext) = $convert_to =~ /pagedimg\_(jpg|gif|png)/i;
-	$convert_to_ext = 'jpg' unless defined $convert_to_ext;
-	$self->{'convert_to_ext'} = $convert_to_ext;
-    }
-
-}
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    $self->SUPER::init($verbosity,$outhandle,$failhandle);
-
-    my $secondary_plugins =  $self->{'secondary_plugins'};
-
-    foreach my $plug_name (keys %$secondary_plugins) {
-	my $plugin = $secondary_plugins->{$plug_name};
-	$plugin->init($verbosity,$outhandle,$failhandle);
-    }
-}
-
-sub deinit {
-    # called only once, after all plugin passes have been done
-
-    my ($self) = @_;
-
-    my $secondary_plugins =  $self->{'secondary_plugins'};
-
-    foreach my $plug_name (keys %$secondary_plugins) {
-	my $plugin = $secondary_plugins->{$plug_name};
-	$plugin->deinit();
-    }
-}
-
-sub convert_post_process
-{
-    # by default do no post processing
-    return;
-}
-
-
-# Run conversion utility on the input file.  
-#
-# The conversion takes place in a collection specific 'tmp' directory so 
-# that we don't accidentally damage the input.
-#
-# The desired output type is indicated by $output_ext.  This is usually
-# something like "html" or "word", but can be "best" (or the empty string)
-# to indicate that the conversion utility should do the best it can.
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename, $textref) = @_;
-    
-    my $outhandle = $self->{'outhandle'};
-    my $convert_to = $self->{'convert_to'};
-    my $failhandle = $self->{'failhandle'};
-    my $convert_to_ext = $self->{'convert_to_ext'};
-    
-
-    my $upgraded_input_filename = &util::upgrade_if_dos_filename($input_filename);
-
-    # derive tmp filename from input filename
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($upgraded_input_filename, "\\.[^\\.]+\$");
-
-    # softlink to collection tmp dir
-    my $tmp_dirname = &util::get_timestamped_tmp_folder();
-    if (defined $tmp_dirname) {
-	$self->{'tmp_dir'} = $tmp_dirname;
-    } else {
-	$tmp_dirname = $dirname;
-    }
-    
-#    # convert to utf-8 otherwise we have problems with the doc.xml file later on
-#    my $utf8_tailname = (&unicode::check_is_utf8($tailname)) ? $tailname : $self->filepath_to_utf8($tailname);
-
-    # make sure filename to be used can be stored OK in a UTF-8 compliant doc.xml file
-     my $utf8_tailname = &unicode::raw_filename_to_utf8_url_encoded($tailname);
-
-
-    # URLEncode this since htmls with images where the html filename is utf8 don't seem
-    # to work on Windows (IE or Firefox), as browsers are looking for filesystem-encoded
-    # files on the filesystem.
-    $utf8_tailname = &util::rename_file($utf8_tailname, $self->{'file_rename_method'}, "without_suffix");
-
-    my $lc_suffix = lc($suffix);
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$utf8_tailname$lc_suffix");
-    
-    # If gsdl is remote, we're given relative path to input file, of the form import/tailname.suffix
-    # But we can't softlink to relative paths. Therefore, we need to ensure that
-    # the input_filename is the absolute path, see http://perldoc.perl.org/File/Spec.html
-    my $ensure_path_absolute = 1; # true
-    &util::soft_link($input_filename, $tmp_filename, $ensure_path_absolute);
-    my $verbosity = $self->{'verbosity'};
-    if ($verbosity > 0) {
-	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
-    }
-
-    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
-    
-    # Execute the conversion command and get the type of the result,
-    # making sure the converter gives us the appropriate output type
-    my $output_type=$self->{'convert_to'};
-#    if ($convert_to =~ m/PagedImage/i) {
-#	$output_type = lc($convert_to)."_".lc($convert_to_ext);
-#    } else {
-#	$output_type = lc($convert_to);
-#    }
-
-    my $cmd = "\"".&util::get_perl_exec()."\" -S gsConvert.pl -verbose $verbosity ";
-    if (defined $self->{'convert_options'}) {
-	$cmd .= $self->{'convert_options'} . " ";
-    }
-    if ($self->{'use_strings'}) {
-      $cmd .= "-use_strings ";
-    }
-    $cmd .= "-errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
-    print STDERR "calling cmd $cmd\n";
-    $output_type = `$cmd`;
-	
-    # remove symbolic link to original file
-    &util::rm($tmp_filename);
-    
-    # Check STDERR here
-    chomp $output_type;
-    if ($output_type eq "fail") {
-	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
-	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
-	# The following  meant that if a conversion failed, the document would be counted twice - do we need it for anything?
-	#$self->{'num_not_processed'} ++;
-	if (-s "$errlog") {
-	    open(ERRLOG, "$errlog");
-	    while (<ERRLOG>) {
-		print $outhandle "$_";
-	    }
-	    print $outhandle "\n";
-	    close ERRLOG;
-	}
-	&util::rm("$errlog") if (-e "$errlog");
-	return "";
-    }
-
-    # store the *actual* output type and return the output filename
-    # it's possible we requested conversion to html, but only to text succeeded
-    #$self->{'convert_to_ext'} = $output_type;
-    if ($output_type =~ /html/i) {
-	$self->{'converted_to'} = "HTML";
-    } elsif ($output_type =~ /te?xt/i) {
-	$self->{'converted_to'} = "Text";
-    } elsif ($output_type =~ /item/i){
-	$self->{'converted_to'} = "PagedImage";
-    }
-    
-    my $output_filename = $tmp_filename;
-    if ($output_type =~ /item/i) {
-	# running under windows
-	if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-	    $output_filename = $tmp_dirname . "\\$tailname\\" . $tailname . ".$output_type";
-	} else {
-	    $output_filename = $tmp_dirname . "\/$tailname\/" . $tailname . ".$output_type";
-	}
-    } else {
-	$output_filename =~ s/$lc_suffix$/.$output_type/;
-    }
-    
-    return $output_filename;
-}
-
-
-# Override BasPlug read_into_doc_obj - we need to call secondary plugin stuff
-sub read_into_doc_obj {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    my $output_ext = $self->{'convert_to_ext'};
-    my $conv_filename = "";
-    $conv_filename = $self->tmp_area_convert_file($output_ext, $filename_full_path);
-        
-    if ("$conv_filename" eq "") {return -1;} # had an error, will be passed down pipeline 
-    if (! -e "$conv_filename") {return -1;} 
-    $self->{'conv_filename'} = $conv_filename;
-    $self->convert_post_process($conv_filename);
-
-    # Run the "fribidi" (http://fribidi.org) Unicode Bidirectional Algorithm program over the converted file
-    # Added for fixing up Persian PDFs after being processed by pdftohtml, but may be useful in other cases too
-    if ($self->{'apply_fribidi'} && $self->{'converted_to'} =~ /(HTML|Text)/) {
-	my $fribidi_command = "fribidi \"$conv_filename\" >\"${conv_filename}.tmp\"";
-	if (system($fribidi_command) != 0) {
-	    print STDERR "ERROR: Cannot run fribidi on \"$conv_filename\".\n";
-	}
-	else {
-	    &util::mv("${conv_filename}.tmp", $conv_filename);
-	}	
-    }
-	
-    my $secondary_plugins =  $self->{'secondary_plugins'};
-    my $num_secondary_plugins = scalar(keys %$secondary_plugins);
-
-    if ($num_secondary_plugins == 0) {
-	print $outhandle "Warning: No secondary plugin to use in conversion.  Skipping $file\n";
-	return 0; # effectively block it
-    }
-
-    my @plugin_names = keys %$secondary_plugins;
-    my $plugin_name = shift @plugin_names;
-	
-    if ($num_secondary_plugins > 1) {
-	print $outhandle "Warning: Multiple secondary plugins not supported yet!  Choosing $plugin_name\n.";
-    }
-   
-    my $secondary_plugin = $secondary_plugins->{$plugin_name};
-
-    # note: metadata is not carried on to the next level
-## **** I just replaced $metadata with {} in following
-    my ($rv,$doc_obj) 
-	= $secondary_plugin->read_into_doc_obj ($pluginfo,"", $conv_filename, $block_hash, {}, $processor, $maxdocs, $total_count, $gli);
-
-    if ((!defined $rv) || ($rv<1)) {
-	# wasn't processed
-	return $rv;
-    }
-    
-    # Override previous gsdlsourcefilename set by secondary plugin
-    my $collect_file = &util::filename_within_collection($filename_full_path);
-    my $collect_conv_file = &util::filename_within_collection($conv_filename);
-    $doc_obj->set_source_filename ($collect_file, $self->{'file_rename_method'}); 
-    ## set_source_filename does not set the doc_obj source_path which is used in archives dbs for incremental
-    # build. so set it manually.
-    $doc_obj->set_source_path($filename_full_path);
-    $doc_obj->set_converted_filename($collect_conv_file);
-
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-        
-    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FileSize", (-s $filename_full_path));
-
-    # ****
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($filename_full_path, "\\.[^\\.]+\$");
-    $doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FilenameRoot", $tailname);
-
-    # do plugin specific processing of doc_obj
-    unless (defined ($self->process($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1;
-    }
-
-    my $topsection = $doc_obj->get_top_section();
-    $self->add_associated_files($doc_obj, $filename_full_path);
-
-    # extra_metadata is already called by sec plugin in process??
-    $self->extra_metadata($doc_obj, $topsection, $metadata); # do we need this here??
-    # do any automatic metadata extraction
-    $self->auto_extract_metadata ($doc_obj);
-
-    # have we found a Title??
-    $self->title_fallback($doc_obj,$topsection,$filename_no_path);
-
-    $self->add_OID($doc_obj);
-
-    return (1, $doc_obj);
-
-}
-
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    return $self->process_type($base_dir, $file, $doc_obj);
-}
-
-# do plugin specific processing of doc_obj for doc_ext type
-sub process_type {
-    my $self = shift (@_);
-    my ($base_dir, $file, $doc_obj) = @_;
-    
-    # need to check that not empty
-    my ($doc_ext) = $file =~ /\.(\w+)$/;
-    $doc_ext = lc($doc_ext);
-    my $file_type = "unknown";
-    $file_type = $self->{'file_type'} if defined $self->{'file_type'};
-    
-    # associate original file with doc object
-    my $cursection = $doc_obj->get_top_section();
-    my $filename = &util::filename_cat($base_dir, $file);
-    my $assocfilename = "doc.$doc_ext";
-    if ($self->{'keep_original_filename'} == 1) {
-	# this should be the same filename that was used for the Source and SourceFile metadata, 
-	# as we will use SourceFile in the srclink (below)
-	$assocfilename = $doc_obj->get_assocfile_from_sourcefile();
-    }
-
-    $doc_obj->associate_file($filename, $assocfilename, undef, $cursection);
-
-    # We use set instead of add here because we only want one value
-    $doc_obj->set_utf8_metadata_element($cursection, "FileFormat", $file_type);
-    my $srclink_filename = "doc.$doc_ext";
-    if ($self->{'keep_original_filename'} == 1) {
-	$srclink_filename = $doc_obj->get_sourcefile();
-    }
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icon".$doc_ext."_"); 
-    $doc_obj->add_utf8_metadata ($cursection, "srclink_file", $srclink_filename);
-    $doc_obj->add_utf8_metadata ($cursection, "srclinkFile", $srclink_filename);
-    return 1;
-}
-
-sub clean_up_after_doc_obj_processing {
-     my $self = shift(@_);
-
-     my $tmp_dir = $self->{'tmp_dir'};
-     if (defined $tmp_dir && -d $tmp_dir) {
-	 ##print STDERR "**** Suppressing clean up of tmp dir\n";
-	 &util::rm_r($tmp_dir);
-	 $self->{'tmp_dir'} = undef;
-     }
-     
-
-}
-1;
-
-
-
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/GreenstoneMETSPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/GreenstoneMETSPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,348 +1,0 @@
-###########################################################################
-#
-# GreenstoneMETSPlugin.pm
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Processes GreenstoneArchive METS documents. Note that this plugin does no
-# syntax checking (though the XML::Parser module tests for
-# well-formedness). It's assumed that the GreenstoneArchive files conform
-# to their DTD.
-
-
-package GreenstoneMETSPlugin;
-
-use Encode;
-use ghtml;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use ReadXMLFile;
-use XML::XPath;
-use XML::XPath::XMLParser;
-
-sub BEGIN {
-    @GreenstoneMETSPlugin::ISA = ('ReadXMLFile');
-}
-
-my $arguments = [ { 'name' => "process_exp",
-		    'desc' => "{BasePlugin.process_exp}",
-		    'type' => "regexp",
-		    'reqd' => "no",
-		    'deft' => &get_default_process_exp()
-		    }
-		  ];
-
-my $options = { 'name'     => "GreenstoneMETSPlugin",
-		'desc'     => "{GreenstoneMETSPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
- 
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)docmets\.xml$^;
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    # have no args - do we still want this?
-    #push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    $self->{'section'} = "";
-    $self->{'section_level'} = 0;
-    $self->{'metadata_name'} = "";
-    $self->{'metadata_value'} = "";
-    $self->{'content'} = "";
-
-    return bless $self, $class;
-}
-
-sub xml_start_document {
-    my $self = shift (@_);
-    my ($expat, $element) =  @_;
-
-    $self->{'section'} = "";
-    $self->{'section_level'} = 0;
-    $self->{'metadata_name'} = "";
-    $self->{'metadata_value'} = "";
-    $self->{'content'} = "";
-
-    #**defined a dmdSection Table
-    $self->{'dmdSec_table'}={};
-
-    #**defined a fileSection Table
-    $self->{'fileSec_table'}={};
-
-    #***open doctxt.xml and read the data in 
-    my $filename = $self->{'filename'};
-    
-    $filename =~ s/docmets.xml$/doctxt.xml/;
-   
-    if (!open (FILEIN, "<:utf8", $filename)) {
-   	print STDERR "Warning: unable to open the $filename\n";
-	$self->{'xmltxt'} = "";
-    }
-    else {
-	my $xml_text = "";
-	while (defined (my $line = <FILEIN>)) {
-	    if ($line !~ m/^<!DOCTYPE.*>/) {
-		$xml_text .= $line;
-	    }
-	}
-
-        my $xml_parser = XML::XPath->new (xml=> $xml_text);
-	#my $xml_tree = $xml_parser->parse ($xml_text);
-
-	#eval {$self->{'parser_text'}->parse};
-	$self->{'parsed_xml'} = $xml_parser;
-    }
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "GreenstoneMETSPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
-    print STDERR "<Processing n='$self->{'file'}' p='GreenstoneMETSPlugin'>\n" if ($self->{'gli'});
-
-}
-
-sub xml_end_document {
-}
-
-sub xml_doctype {
-}
-
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-    $self->{'element'} = $element;
-    #**deal with dmdSection
-    if ($element =~ /^(mets:)?dmdSec$/ || $element =~ /(gsdl3:)?Metadata$/){ 
-	$self->xml_dmd_start_tag (@_);
-    } elsif ($element =~ /^(mets:)?file$/) {
-	# only store the file_id for sections with text. Not for default ids (assoc files)
-	if ($_{'ID'} =~ m/FILE(.*)/) {
-	    $self->{'file_Id'} = $1;
-	}
-	else {
-	    undef $self->{'file_Id'};
-	}
-    } elsif ($element =~ /^(mets:)?FLocat$/){
-	#***deal with fileSection
-	$self->xml_fileloc_start_tag (@_);
-    } elsif ($element =~ /^(mets:)?div$/){
-	#***deal with StrucMap Section
-	$self->xml_strucMap_start_tag (@_);
-    }
-}
-
-sub xml_dmd_start_tag {
-    my $self = shift (@_);
-    my ($expat, $element) = @_;
-
-    if ($element =~ /^(mets:)?dmdSec$/){
-  	my ($section_num) = ($_{'ID'} =~ m/DM(.*)/);
-	$self->{'dmdSec_table'}->{"$section_num"}=[];
-	$self->{'dmdSec_table'}->{'section_num'}=$section_num;
-    } elsif ($element =~ /^(gsdl3:)?Metadata$/) {
-	$self->{'metadata_name'} = $_{'name'};
-    }
-}
-
-sub xml_fileloc_start_tag {
-    my $self = shift (@_);
-    my ($expat, $element) = @_;
-
-    my $xlink = $_{'xlink:href'};
-    if (!defined $xlink) {
-	# try without namespace
-	$xlink = $_{'href'};
-    }
-    #my ($section_num) = ($_{'ID'} =~ m/^FLOCAT(.*)$/);
-    my $section_num = $self->{'file_Id'};
-    return if (!defined $section_num); 
-
-    $self->{'fileSec_table'}->{"$section_num"}=[];
-    $self->{'fileSec_table'}->{'section_num'}=$section_num;
-    
-    my ($filename,$xpath_expr)=($xlink =~ m/^file:(.*)\#xpointer\((.*)\)$/);
-    my $nodeset = $self->{'parsed_xml'}->findnodes ($xpath_expr);
-    my $node_size= $nodeset->size;
-    
-    if ($node_size==0) {
-	print STDERR "Warning: no text associated with XPATH $xpath_expr\n";
-    }
-    else {
-	foreach my $node ($nodeset->get_nodelist) {
-	    my $xml_content = XML::XPath::XMLParser::as_string($node);
-	    my $unescaped_xml_content = &ghtml::unescape_html($xml_content);
-	    my $section_content={'section_content'=> $unescaped_xml_content};
-	    
-	    my $content_list = $self->{'fileSec_table'}->{"$section_num"};
-	    push (@$content_list, $section_content);
-	}
-    }
-}
-
-sub xml_strucMap_start_tag {
-    my $self = shift (@_);
-    my ($expat, $element) = @_;
-
-
-    my ($section_num) = ($_{'ID'} =~ m/DS(.*)/);
-
-    if ($_{'ID'} ne "DSAll"){
-	if ($self->{'section_level'}==0) {
-	    $self->open_document();
-	} else {
-	    my $doc_obj = $self->{'doc_obj'};
-	    $self->{'section'}=
-		$doc_obj->insert_section($doc_obj->get_end_child($self->{'section'}));
-	}
-	$self->{'section_level'}++;
-	
-	#***Add metadata from dmdSection
-	my $md_list = $self->{'dmdSec_table'}->{"$section_num"};
-	
-	foreach my $md_pair (@$md_list){
-	    # text read in by XML::Parser is in Perl's binary byte value
-	    # form ... need to explicitly make it UTF-8
-
-	    my $metadata_name = decode("utf8",$md_pair->{'metadata_name'});
-	    my $metadata_value = decode("utf8",$md_pair->{'metadata_value'});
-
-	    $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, 
-						  $metadata_name, $metadata_value);
-	}
-	
-	#*** Add content from fileSection
-	my $content_list = $self->{'fileSec_table'}->{"$section_num"};
-	
-	foreach my $section_content (@$content_list){
-	    # Don't need to decode $content as this has been readin in
-	    # through XPath which (unlike XML::Parser) correctly sets
-	    # the string to be UTF8 rather than a 'binary' string of bytes
-	    my $content = $section_content->{'section_content'};
-
-	    $self->{'doc_obj'}->add_utf8_text($self->{'section'},$content);
-	}
-    }
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "mets:mets";
-}
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    if ($element =~ /^(gsdl3:)?Metadata$/) {
-	my $section_num = $self->{'dmdSec_table'}->{'section_num'};
-	my $metadata_name=$self->{'metadata_name'};
-	my $metadata_value=$self->{'metadata_value'};
-
-	my $md_pair={'metadata_name' => $metadata_name,
-		     'metadata_value'=> $metadata_value};
-
-	my $md_list = $self->{'dmdSec_table'}->{"$section_num"};
-
-	push(@$md_list,$md_pair);
-      
-	$self->{'metadata_name'} = "";
-	$self->{'metadata_value'} = "";
-    } elsif ($element =~ /^(mets:)?file$/){
-	$self->{'file_id'} = "";
-    }
-    
-    
-    #*** StrucMap Section
-    if ($element =~ /^(mets:)?div$/) {
-	$self->{'section_level'}--;
-	$self->{'section'} = $self->{'doc_obj'}->get_parent_section($self->{'section'});
-	$self->close_document() if $self->{'section_level'}==0;
-    }
-    $self->{'element'} = "";
-}
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_; 
-
-    if ($self->{'element'} =~ /^(gsdl3:)?Metadata$/) {
-   	$self->{'metadata_value'} .= $_;
-    }
-}
-
-sub open_document {
-    my $self = shift(@_);
-
-    # create a new document
-    $self->{'doc_obj'} = new doc ();
-    $self->{'section'} = "";
-}
-
-sub close_document {
-    my $self = shift(@_);
-    
-    # add the associated files
-    my $assoc_files = 
-	$self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
-
-    # for when "assocfilepath" isn't the same directory that doc.xml is in...
-    my $assoc_filepath_list= $self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "assocfilepath");
-
-    my $assoc_filepath=shift (@$assoc_filepath_list);
-    if (defined ($assoc_filepath)) {
-	# make absolute rather than relative...
-	$self->{'filename'} =~ m@^(.*[\\/]archives)@;
-	$assoc_filepath = "$1/$assoc_filepath/";
-    } else {
-	$assoc_filepath = $self->{'filename'};
-	$assoc_filepath =~ s/[^\\\/]*$//;
-    }
-
-    foreach my $assoc_file_info (@$assoc_files) {
-	my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
-	my $real_dir = &util::filename_cat($assoc_filepath, $assoc_file),
-	my $assoc_dir = (defined $dir && $dir ne "") 
-	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
-	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
-    }
-    $self->{'doc_obj'}->delete_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
-
-    # process the document
-    $self->{'processor'}->process($self->{'doc_obj'}, $self->{'file'});
-}
-
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/HTMLImagePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/HTMLImagePlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,1076 +1,0 @@
-###########################################################################
-#
-# HTMLImagePlugin.pm -- Context-based image indexing plugin for HTML documents
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# DESCRIPTION:
-#
-#  Extracts images and associated text and metadata from
-#  web pages as individual documents for indexing. Thumbnails
-#  are created from each image for browsing purposes.
-#
-#  Options are available for configuring the aggressiveness of the
-#  associated text extraction mechanisms. A higher level of
-#  aggressiveness will extract more text and consequently
-#  may mean lower accuracy (precision); however, it may also 
-#  retrieve more of the relevant images from the collection (recall). 
-#  Lower levels of aggressiveness maybe result in slightly faster
-#  collection builds at the import stage.
-#
-#  HTMLImagePlugin is a subclass of HTMLPlug (i.e. it will index pages also 
-#  if required). It can be used in place of HTMLPlugin to index both
-#  pages and their images.
-#
-# REQUIREMENTS:
-#   
-#  The ImageMagick image manipulation is used to create
-#  thumbnails and extract some image metadata. (Available 
-#  from http://www.imagemagick.org/)
-#
-#  Unix:
-#    Many Linux distributions contain ImageMagick.
-#
-#  Windows:
-#    ImageMagick can be downloaded from the website above.
-#    Make sure the system path includes the ImageMagick binaries
-#    before using HTMLImagePlugin.
-#
-#    NOTE: NT/2000/XP contain a filesystem utility 'convert.exe' 
-#    with the same name as the image conversion utility. The
-#    ImageMagick FAQ recommends renaming the filesystem
-#    utility (e.g. to 'fsconvert.exe') to avoid this clash.
-#
-# USAGE:  
-#
-#  An image document consists of metadata elements:
-#
-#   OriginalFilename, FilePath, Filename, FileExt, FileSize,
-#   Width, Height, URL, PageURL, ThumbURL, CacheURL, CachePageURL
-#   ImageText, PageTitle
-#
-#  Most of these are only useful in format strings (e.g. ThumbURL, 
-#  Filename, URL, PageURL, CachePageURL). 
-#
-#  ImageText, as the name suggests contains the indexable text.
-#  (unless using the -document_text plugin option)
-#
-#  Since image documents are made up of metadata elements 
-#  alone, format strings are needed to display them properly. 
-#  NOTE: The receptionist will only display results (e.g. thumbnails)
-#  in 4 columns if the format string begins with "<td><table>".
-#
-#  The example below takes the user to the image within the
-#  source HTML document rather than using a format string
-#  on DocumentText to display the image document itself.
-#
-#  Example collect.cfg:
-#
-#   ...
-#
-#   indexes document:ImageText document:text
-#   defaultindex document:ImageText
-#  
-#   collectionmeta .document:ImageText "images"
-#   collectionmeta .document:text "documents"
-#
-#   ...
-#
-#   plugin HTMLImagePlugin -index_pages -aggressiveness 6
-#
-#   ...
-#  
-#   format SearchVList '<td>{If}{[Title],[link][icon]&nbsp;[Title][[/link],
-#    <table><tr><td align="center"><a href="[CachePageURL]">
-#    <img src="[ThumbURL]"></a></td></tr><tr><td align="center">
-#    <a href="[CachePageURL]"><font size="-1">[OriginalFilename]</font></a>
-#    <br>[Width]x[Height]</td></tr></table>}</td>'
-#
-#   ...
-#
- 
-package HTMLImagePlugin;
-
-use HTMLPlugin;
-use ghtml;
-use unicode;
-use util;
-use strict; # 'subs';
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @HTMLImagePlugin::ISA = qw( HTMLPlugin );
-}
-
-my $aggressiveness_list = 
-    [ { 'name' => "1",
-	'desc' => "{HTMLImagePlugin.aggressiveness.1}" },
-      { 'name' => "2",
-	'desc' => "{HTMLImagePlugin.aggressiveness.2}" },
-      { 'name' => "3",
-	'desc' => "{HTMLImagePlugin.aggressiveness.3}" },
-      { 'name' => "4",
-	'desc' => "{HTMLImagePlugin.aggressiveness.4}" },
-      { 'name' => "5",
-	'desc' => "{HTMLImagePlugin.aggressiveness.5}" },
-      { 'name' => "6",
-	'desc' => "{HTMLImagePlugin.aggressiveness.6}" },
-      { 'name' => "7",
-	'desc' => "{HTMLImagePlugin.aggressiveness.7}" },
-      { 'name' => "8",
-	'desc' => "{HTMLImagePlugin.aggressiveness.8}" },
-      { 'name' => "9",
-	'desc' => "{HTMLImagePlugin.aggressiveness.9}" } ];
-
-my $arguments =
-    [ { 'name' => "aggressiveness",
-	'desc' => "{HTMLImagePlugin.aggressiveness}",
-	'type' => "int",
-	'list' => $aggressiveness_list,
-	'deft' => "3",
-	'reqd' => "no" },
-      { 'name' => "index_pages",
-	'desc' => "{HTMLImagePlugin.index_pages}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "no_cache_images",
-	'desc' => "{HTMLImagePlugin.no_cache_images}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "min_size",
-	'desc' => "{HTMLImagePlugin.min_size}",
-	'type' => "int",
-	'deft' => "2000",
-	'reqd' => "no" },
-      { 'name' => "min_width",
-	'desc' => "{HTMLImagePlugin.min_width}",
-	'type' => "int",
-	'deft' => "50",
-	'reqd' => "no" },
-      { 'name' => "min_height",
-	'desc' => "{HTMLImagePlugin.min_height}",
-	'type' => "int",
-	'deft' => "50",
-	'reqd' => "no" },
-      { 'name' => "thumb_size",
-	'desc' => "{HTMLImagePlugin.thumb_size}",
-	'type' => "int",
-	'deft' => "100",
-	'reqd' => "no" },
-      { 'name' => "convert_params",
-	'desc' => "{HTMLImagePlugin.convert_params}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "min_near_text",
-	'desc' => "{HTMLImagePlugin.min_near_text}",
-	'type' => "int",
-	'deft' => "10",
-	'reqd' => "no" },
-      { 'name' => "max_near_text",
-	'desc' => "{HTMLImagePlugin.max_near_text}",
-	'type' => "int",
-	'deft' => "400",
-	'reqd' => "no" },
-      { 'name' => "smallpage_threshold",
-	'desc' => "{HTMLImagePlugin.smallpage_threshold}",
-	'type' => "int",
-	'deft' => "2048",
-	'reqd' => "no" },
-      { 'name' => "textrefs_threshold",
-	'desc' => "{HTMLImagePlugin.textrefs_threshold}",
-	'type' => "int",
-	'deft' => "2",
-	'reqd' => "no" },
-      { 'name' => "caption_length",
-	'desc' => "{HTMLImagePlugin.caption_length}",
-	'type' => "int",
-	'deft' => "80",
-	'reqd' => "no" },
-      { 'name' => "neartext_length",
-	'desc' => "{HTMLImagePlugin.neartext_length}",
-	'type' => "int",
-	'deft' => "300",
-	'reqd' => "no" },
-      { 'name' => "document_text",
-	'desc' => "{HTMLImagePlugin.document_text}",
-	'type' => "flag",
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "HTMLImagePlugin",
-		'desc'     => "{HTMLImagePlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new HTMLPlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    # init class variables
-    $self->{'textref'} = undef; # init by read_file fn
-    $self->{'htdoc_obj'} = undef; # init by process fn
-    $self->{'htpath'} = undef; # init by process fn
-    $self->{'hturl'} = undef; # init by process fn
-    $self->{'plaintext'} = undef; # HTML stripped version - only init if needed by raw_neartext sub
-    $self->{'smallpage'} = 0; # set by process fn
-    $self->{'images_indexed'} = undef; # num of images indexed - if 1 or 2 then we know page is small
-    $self->{'initialised'} = undef; # flag (see set_extraction_options())
-
-    return bless $self, $class;
-}
-
-# if indexing pages, let HTMLPlugin do it's stuff
-# image extraction done through read()
-sub process {
-    my $self = shift(@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    $self->{'imglist'} = ();
-    if ( $self->{'index_pages'} ) {
-	my $ok = $self->SUPER::process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli);
-	if ( ! $ok ) { return $ok }
-	$self->{'htdoc_obj'} = $doc_obj;
-    }
-    # else use URL for referencing
-    #if ( $file =~ /(.*)[\/\\]/ ) { $self->{'htpath'} = $1; } else { $self->{'htpath'} = $file; }
-
-    $self->{'htpath'} = $base_dir if (-d $base_dir);
-    if ( $file =~ /(.*)[\/\\]/ ) { $self->{'htpath'} .= "/$1"; }
-    $self->{'htpath'} =~ s/\\/\//g;  # replace \ with /
-
-    $self->{'hturl'} = "http://$file";
-    $self->{'hturl'} =~ s/\\/\//g; # for windows
-    ($self->{'filename'}) = $file =~ /.*[\/\\](.*)/;
-    ($self->{'base_path'}) = $file =~ /(.*)[\/\\]/i;
-    if ( ( -s "$base_dir/$file") <= $self->{'smallpage_threshold'} ) {
-	$self->{'smallpage'} = 1;
-    } else { $self->{'smallpage'} = 0; }
-
-    if ( defined($self->{'initialised'}) ) { return 1; }
-    else {
-	$self->{'initialised'} = $self->set_extraction_options($base_dir =~ /^(.*?)\/import/i);
-	return $self->{'initialised'};
-    }
-}
-
-# get complex configuration options from configuration files
-# -- $GSDLCOLLECTION/etc/HTMLImagePlugin.cfg (tag sets for aggr 2+)
-# -- $GSDLHOME/etc/packages/phind/stopword/en/brown.sw (stopwords for aggr 5+)
-
-# If there's no HTMLImagePlugin.cfg file we'll use the following default values
-my $defaultcfg = '
-<delimitertagset>
-  <setname>Caption</setname>
-  <taggroup>font</taggroup>
-  <taggroup>tt</taggroup>
-  <taggroup>small</taggroup>
-  <taggroup>b</taggroup>
-  <taggroup>i</taggroup>
-  <taggroup>u</taggroup>
-  <taggroup>em</taggroup>
-  <taggroup>td</taggroup>
-  <taggroup>li</taggroup>
-  <taggroup>a</taggroup>
-  <taggroup>p</taggroup>
-  <taggroup>tr</taggroup>
-  <taggroup>center</taggroup>
-  <taggroup>div</taggroup>
-  <taggroup>caption</taggroup>
-  <taggroup>br</taggroup>
-  <taggroup>ul</taggroup>
-  <taggroup>ol</taggroup>
-  <taggroup>table</taggroup>
-  <taggroup>hr</taggroup>
-</delimitertagset>
-
-<delimitertagset>
-  <setname>Neartext</setname>
-  <taggroup>tr|hr|table|h\d|img|body</taggroup>
-  <taggroup>td|tr|hr|table|h\d|img|body</taggroup>
-  <taggroup>p|br|td|tr|hr|table|h\d|img|body</taggroup>
-  <taggroup>font|p|i|b|em|img</taggroup>
-</delimitertagset>
-';
-
-sub set_extraction_options() {
-    my ($self, $collpath) = @_;
-    my ($filepath);
-
-    print {$self->{'outhandle'}} "HTMLImagePlugin: Initialising\n"
-	if $self->{'verbosity'} > 1;
-    # etc/HTMLImagePlugin.cfg (XML)
-    # tag sets for captions and neartext
-    if ( $self->{'aggressiveness'} > 1 && $self->{'aggressiveness'} != 9 ) {
-	$self->{'delims'} = [];
-	$self->{'cdelims'} = [];
-	my ($cfg, @tagsets, $tagset, $type, @delims);
-
-	$filepath = "$collpath/etc/HTMLImagePlugin.cfg";
-	if ( open CFG, "<$filepath" ) {
-	    while (<CFG>) { $cfg .= $_ }
-	    close CFG;
-	} else {
-	    $cfg = $defaultcfg;
-	}
-
-	(@tagsets) = 
-	    $cfg =~ /<delimitertagset>(.*?)<\/delimitertagset>/igs;
-	foreach $tagset ( @tagsets ) {
-	    ($type) = $tagset =~ /<setname>(.*?)<\/setname>/i;
-	    if ( lc($type) eq "caption" ) {
-		(@{$self->{'cdelims'}}) = $tagset =~ /<taggroup>(.*?)<\/taggroup>/igs;
-	    }
-	    elsif ( lc($type) eq "neartext" ) {
-		(@{$self->{'delims'}}) = $tagset =~ /<taggroup>(.*?)<\/taggroup>/igs;
-	    }
-	}
-
-	# output a warning if there seem to be no delimiters
-	if ( scalar(@{$self->{'cdelims'}} == 0)) {
-	    print {$self->{'outhandle'}} "HTMLImagePlugin: Warning: no caption delimiters found in $filepath\n";
-	}
-	if ( scalar(@{$self->{'delims'}} == 0)) {
-	    print {$self->{'outhandle'}} "HTMLImagePlugin: Warning: no neartext delimiters found in $filepath\n";
-	}
-    }
-    
-    # get stop words for textual reference extraction
-    # TODO: warnings scroll off. Would be best to output them again at end of import
-    if ( $self->{'aggressiveness'} >=5 && $self->{'aggressiveness'} != 9 ) {
-	$self->{'stopwords'} = ();
-	$filepath = &util::filename_cat($ENV{'GSDLHOME'}, "etc", "packages", "phind", "stopword", "en", "brown.sw");
-	if ( open STOPWORDS, "<$filepath" ) {
-	    while ( <STOPWORDS> ) {
-		chomp;
-		$self->{'stopwords'}{$_} = 1;
-	    }
-	    close STOPWORDS;
-	} else {
-	    print {$self->{'outhandle'}} "HTMLImagePlugin: Warning: couldn't open stopwords file at $filepath ($!)\n";
-	}
-	
-    }
-
-    if ( $self->{'neartext_length'} > $self->{'max_near_text'} ) {
-	$self->{'max_near_text'} = $self->{'neartext_length'} * 1.33;
-	print {$self->{'outhandle'}} "HTMLImagePlugin: Warning: adjusted max_text to $self->{'max_near_text'}\n";
-    } 
-    if ( $self->{'caption_length'} > $self->{'max_near_text'} ) {
-	$self->{'max_near_text'} = $self->{'caption_length'} * 1.33;
-	print {$self->{'outhandle'}} "HTMLImagePlugin: Warning: adjusted max_text to $self->{'max_near_text'}\n";
-    }
-
-    return 1;
-}
-
-# return number of files processed, undef if can't recognise, -1 if 
-# cant process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my ($self, $pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = (@_);
-    my ($doc_obj, $section, $filepath, $imgtag, $pos, $context, $numdocs, $tndir, $imgs);
-    # forward normal read (runs HTMLPlugin if index_pages T)
-    my $ok =  $self->SUPER::read($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli); 
-    if ( ! $ok ) { return $ok } # what is this returning??
-
-    my $outhandle = $self->{'outhandle'};
-    my $textref = $self->{'textref'};
-    my $htdoc_obj = $self->{'htdoc_obj'};
-    $numdocs = 0;
-    $base_dir =~ /(.*)\/.*/;
-    $tndir = "$1/archives/thumbnails"; # TODO: this path shouldn't be hardcoded?
-    &util::mk_all_dir($tndir) unless -e "$tndir"; 
-
-    $imgs = \%{$self->{'imglist'}};
-    my $nimgs = $self->get_img_list($textref);
-    $self->{'images_indexed'} = $nimgs;
-    if ( $nimgs > 0 ) {
-	my @fplist = (sort { $imgs->{$a}{'pos'} <=> $imgs->{$b}{'pos'} } keys %{$imgs});
-	my $i = 0;
-	foreach $filepath ( @fplist ) {
-	    $pos = $imgs->{$filepath}{'pos'}; 
-	    $context = substr ($$textref, $pos - 50, $pos + 50); # grab context (quicker)
-	    ($imgtag) = ($context =~ /(<(?:img|a|body)\s[^>]*$filepath[^>]*>)/is );
-	    if (! defined($imgtag)) { $imgtag = $filepath }
-	    print $outhandle "HTMLImagePlugin: extracting $filepath\n"
-		if ( $self->{'verbosity'} > 1 );
-	    $doc_obj = new doc ("", "indexed_doc", $self->{'file_rename_method'});
-	    $section = $doc_obj->get_top_section();
-	    my $prevpos = ( $i == 0 ? 0 : $imgs->{$fplist[$i - 1]}{'pos'});
-	    my $nextpos = ( $i >= ($nimgs -1) ? -1 : $imgs->{$fplist[$i + 1]}{'pos'} );
-
-	    $self->extract_image_info($imgtag, $filepath, $textref, $doc_obj, $section, $tndir, $prevpos, $nextpos);
-            $processor->process($doc_obj);
-	    $numdocs++;
-	    $i++;
-	}
-	return $numdocs;
-    } else {
-	print $outhandle "HTMLImagePlugin: No images from $file indexed\n"
-	    if ( $self->{'verbosity'} > 2 );
-	return 1;
-    }
-    
-}
-
-# for every valid image tag
-# 1. extract related text and image metadata
-# 2. add this as document meta-data
-# 3. add assoc image(s) as files
-#
-sub extract_image_info {
-    my $self = shift (@_);
-    my ($tag, $id, $textref, $doc_obj, $section, $tndir, $prevpos, $nextpos) = (@_);
-    my ($filename, $orig_fp, $fn, $ext, $reltext, $relreltext, $crcid, $imgs,
-	$thumbfp, $pagetitle, $alttext,	$filepath, $aggr);
-
-    my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
-
-    $aggr = $self->{'aggressiveness'};
-    $imgs = \%{$self->{'imglist'}};
-    $filepath = $imgs->{$id}{'relpath'}; 
-    ($filename) = $filepath =~ /([^\/\\]+)$/s;
-    ($orig_fp) = "$self->{'base_path'}/$filepath"; 
-    $orig_fp =~ tr/+/ /;
-    $orig_fp =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; # translate %2E to space, etc
-    $orig_fp =~ s/\\/\//g;
-    $filepath = "$self->{'htpath'}/$filepath";
-    my ($onlyfn) = $filename =~ /([^\\\/]*)$/;
-    ($fn, $ext) = $onlyfn =~ /(.*)\.(.*)/;
-    $fn = lc $fn; $ext = lc $ext;
-    ($reltext) = "<tr><td>GifComment</td><td>" . `$imagick_cmd identify $filepath -ping -format "%c"` . "</td></tr>\n"
-        if ($ext eq "gif");
-    $reltext .= "<tr><td>FilePath</td><td>$orig_fp</td></tr>\n";
-
-    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-	$crcid = "$fn.$ext." . $self->{'next_crcid'}++;
-    } else { 
-	($crcid) = `cksum $filepath` =~ /^(\d+)/; 
-    }
-    
-    $thumbfp = "$tndir/tn_$crcid.jpg";
-    `$imagick_cmd convert -flatten -filter Hanning $self->{'convert_params'} -geometry "$self->{'thumb_size'}x$self->{'thumb_size'}>" $filepath $thumbfp` unless -e $thumbfp;
-    if ( ! (-e $thumbfp) ) {
-	print STDERR "HTMLImagePlugin: 'convert' failed. Check ImageMagicK binaries are installed and working correctly\n"; return 0; 
-    }
-    
-    # shove in full text (tag stripped or unstripped) if settings require it
-    if ( $aggr == 10) {
-	$reltext = "<tr><td>AllPage</td><td>" . $$textref . "</td><tr>\n";   # level 10 (all text, verbatim)
-    } else {
-	$pagetitle = $self->get_meta_value("title", $textref);
-	($alttext) = $tag =~ /\salt\s*=\s*(?:\"|\')(.+?)(?:\"|\')/is;
-	if ( defined($alttext) && length($alttext) > 1) {
-	    $reltext .= "<tr><td>ALTtext</td><td>$alttext</td></tr>\n"; }
-	$reltext .= "<tr><td>SplitCapitalisation</td><td>" . 
-	    $self->split_filepath($orig_fp) . "</td></tr>\n";
-
-	# get caption/tag based near text (if appropriate)
-	if ( $aggr > 1 ) {
-	    if ( $aggr >= 2 ) {
-		$reltext .= 
-		    $self->extract_caption_text($tag, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos);
-		$relreltext = $reltext;
-	    } 
-	    # repeat the filepath, alt-text, caption, etc
-	    if ( $aggr == 8 ) {
-		$reltext .= $relreltext; 
-	    }
-	    if ( $aggr >= 3 ) {
-		$reltext .= 
-		    $self->extract_near_text($tag, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos);
-	    }
-	
-	    # get page metadata (if appropriate)
-	    if ( $aggr >= 6 || ( $aggr >= 2 && 
-						     ( $self->{'images_indexed'} < 2 || 
-						       ($self->{'smallpage'} == 1 && $self->{'images_indexed'} < 6 )))) {	
-		$reltext .= $self->get_page_metadata($textref);
-	    }
-	    # textual references
-	    if ( $aggr  == 5 || $aggr >= 7) {
-		if ( length($relreltext) > ($self->{'caption_length'} * 2) )  {
-		    $reltext .= $self->get_textrefs($relreltext, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos); }
-		else {
-		    $reltext .= $self->get_textrefs($reltext, $textref, $prevpos, $imgs->{$id}{'pos'}, $nextpos); 
-		}
-	    }
-	} # aggr > 1
-    } # aggr != 10
-    
-    $doc_obj->set_OID($crcid); 
-    $doc_obj->associate_file($thumbfp, "$fn.thumb.jpg", undef, $section);
-    $doc_obj->add_metadata($section, "OriginalFilename", $filename);
-    $doc_obj->add_metadata($section, "FilePath", $orig_fp);
-    $doc_obj->add_metadata($section, "Filename", $fn);
-    $doc_obj->add_metadata($section, "FileExt", $ext);
-    $doc_obj->add_metadata($section, "FileSize", $imgs->{$id}{'filesize'});
-    $doc_obj->add_metadata($section, "Width", $imgs->{$id}{'width'});
-    $doc_obj->add_metadata($section, "Height", $imgs->{$id}{'height'});
-    $doc_obj->add_metadata($section, "URL", "http://$orig_fp");
-    $doc_obj->add_metadata($section, "PageURL", $self->{'hturl'});
-    $doc_obj->add_metadata($section, "PageTitle", $pagetitle);
-    $doc_obj->add_metadata($section, "ThumbURL", 
-			   "_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/$fn.thumb.jpg");
-    $doc_obj->add_metadata($section, "FileFormat", "W3Img");
-
-    if ( $self->{'document_text'} ) {
-	$doc_obj->add_utf8_text($section, "<table border=1>\n$reltext</table>");
-    } else {
-	$doc_obj->add_metadata($section, "ImageText", "<table border=1>\n$reltext</table>\n");
-    }
-    
-    if ( $self->{'index_pages'} ) {
-	my ($cache_url) = "_httpdoc_&d=" . $self->{'htdoc_obj'}->get_OID();
-	if ( $imgs->{$id}{'anchored'} ) {
-	    my $a_name = $id;
-	    $a_name =~ s/[\/\\\:\&]/_/g;
-	    $cache_url .=  "#gsdl_$a_name" ;
-	}
-	$doc_obj->add_utf8_metadata($section, "CachePageURL", $cache_url);
-    }
-    if ( ! $self->{'no_cache_images'} ) {
-	$onlyfn = lc $onlyfn;
-	$doc_obj->associate_file($filepath, $onlyfn, undef, $section);
-	$doc_obj->add_utf8_metadata($section, "CacheURL", 
-			       "_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/$onlyfn");
-    }
-    return 1;
-}
-
-sub get_page_metadata {
-    my ($self, $textref) = (@_);
-    my (@rval);
-    $rval[0] = $self->get_meta_value("title", $textref);
-    $rval[1] = $self->get_meta_value("keywords", $textref);
-    $rval[2] = $self->get_meta_value("description", $textref);
-    $rval[3] = $self->{'filename'};
-
-    return wantarray ? @rval : "<tr><td>PageMeta</td><td>@rval</td></tr>\n" ;
-}
-
-# turns LargeCatFish into Large,Cat,Fish so MG sees the separate words
-sub split_filepath {
-    my ($self, $filepath) = (@_);
-    my (@words) = $filepath =~ /([A-Z][a-z]+)/g; 
-    return join(',', @words);
-}
-
-# finds and extracts sentences
-# that seem to be on the same topic
-# as other related text (correlations)
-# and textual references (e.g. in figure 3 ...)
-sub get_textrefs {
-    my ($self, $reltext, $textref, $prevpos, $pos, $nextpos) = (@_);
-    my ($maxtext, $mintext, $startpos, $context_size, $context);
-
-    my (@relwords, @refwords, %sentences, @pagemeta);
-
-    # extract larger context
-    $maxtext = $self->{'max_near_text'};
-    $startpos = $pos - ($maxtext * 4);
-    $context_size = $maxtext*10;
-    if ($startpos < $prevpos ) { $startpos = $prevpos }
-    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) { $context_size = ($nextpos - $startpos) }
-    $context = substr ( $$textref, $startpos, $context_size );
-    $context =~ s/<.*?>//gs;
-    $context =~ s/^.*>(.*)/$1/gs;
-    $context =~ s/(.*)<.*$/$1/gs;
-
-    # get page meta-data (if not already included)
-    if ( $self->{'aggressiveness'} == 5 && ! $self->{'smallpage'} ) {
-	@pagemeta = $self->get_page_metadata($textref);
-	foreach my $value ( @pagemeta ) {
-	    $context .= "$value."; # make each into psuedo-sentence
-	}
-    }
-
-    # TODO: this list is not exhaustive
-    @refwords = ( '(?:is|are)? ?(?:show(?:s|n)|demonstrate(?:d|s)|explains|features) (?:in|by|below|above|here)', 
-		  '(?:see)? (?:figure|table)? (?:below|above)');    
-
-    # extract general references
-    foreach my $rw ( @refwords ) {
-	while ( $context =~ /[\.\?\!\,](.*?$rw\W.*?[\.\?\!\,])/ig ) {
-	    my $sentence = $1;
-	    $sentence =~ s/\s+/ /g;
-	    $sentences{$sentence}+=2;
-	}
-    }
-    # extract specific (figure, table) references by number
-    my ($fignum) = $context =~ /[\.\?\!].*?(?:figure|table)s?[\-\_\ \.](\d+\w*)\W.*?[\.\?\!]/ig;
-    if ( $fignum ) {
-	foreach my $rw ( @refwords ) {
-	    while ( $context =~ /[\.\?\!](.*?(figure|table)[\-\_\ \.]$fignum\W.*?[\.\?\!])/ig ) {
-		my $sentence = $1;
-		$sentence =~ s/\s+/ /g;
-		$sentences{$sentence}+=4;
-	    }
-	}
-    }
-
-    # sentences with occurances of important words
-    @relwords = $reltext =~ /([a-zA-Z]{4,})/g; # take out small words
-    foreach my $word ( @relwords ) {
-	if ( $self->{'stopwords'}{$word} ) { next } # skip stop words
-	while ( $context =~ /([^\.\?\!]*?$word\W.*?[\.\?\!])/ig ) {
-	    my $sentence = $1;
-	    $sentence =~ s/\s+/ /g;
-	    $sentences{$sentence}++;
-	}
-    }
-    foreach my $sentence ( keys %sentences ) {
-	if ($sentences{$sentence} < $self->{'textrefs_threshold'}) {
-	    delete $sentences{$sentence};
-	}
-    }
-    my ($rval) = join "<br>\n", (keys %sentences);
-    if ( $rval && length($rval) > 5 ) {
-	return ( "<tr><td>TextualReferences</td><td>" . $rval . "</td></tr>\n") }
-    else { return "" }
-}
-
-# handles caption extraction
-# calling the extractor with different
-# tags and choosing the best candidate caption
-sub extract_caption_text {
-    my ($self, $tag, $textref, $prevpos, $pos, $nextpos) = (@_);
-    my (@neartext, $len, $hdelim, $mintext, $goodlen,
-	$startpos, $context, $context_size);
-    
-    $mintext = $self->{'min_near_text'};
-    $goodlen = $self->{'caption_length'};
-
-    # extract a context to extract near text from (faster)
-    $context_size = $self->{'max_near_text'}*3;
-    $startpos = $pos - ($context_size / 2);
-    if ($startpos < $prevpos ) { $startpos = $prevpos }
-    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) 
-    { $context_size = ($nextpos - $startpos) }
-
-    $context = substr ( $$textref, $startpos, $context_size );
-    $context =~ s/<!--.*?-->//gs;
-    $context =~ s/^.*-->(.*)/$1/gs;
-    $context =~ s/(.*)<!--.*$/$1/gs;
-
-    # try stepping through markup delimiter sets
-    # and selecting the best one
-    foreach $hdelim ( @{ $self->{'cdelims'} } ) {
-	@neartext = $self->extract_caption($tag, $hdelim, \$context);
-	$len = length(join("", @neartext));
-	last if ($len >= $mintext && $len <= $goodlen);
-    }
-    # reject if well over reasonable length
-    if ( $len > $goodlen ) {
-	@neartext = [];
-    }
-    $neartext[0] = " " if (! defined $neartext[0]);
-    $neartext[1] = " " if (! defined $neartext[1]);
-    return "<tr><td>Caption</td><td>" . (join ",",  @neartext) . "</td></tr>\n"; # TODO: the | is for testing purposes
-} # end extract_caption_text
-
-# the previous section header often gives a bit
-# of context to the section that the image is
-# in (invariably the header is before/above the image)
-# so extract the text of the closest header above the image
-#
-# this fn just gets all the headers above the image, within the context window
-sub get_prev_header {
-    my ($self, $pos, $textref) = (@_);
-    my ($rhtext);
-    while ( $$textref =~ /<h\d>(.*?)<\/h\d>/sig ) {
-	# only headers before image
-	if ((pos $$textref) < $pos) {
-	    $rhtext .= "$1, ";
-	}
-    } 
-    if ( $rhtext ) { return "Header($rhtext)" }
-    else { return "" }
-}
-
-# not the most robust tag stripping 
-# regexps (see perl.com FAQ) but good enough
-#
-# used by caption & tag-based near text algorithms
-sub strip_tags {
-    my ( $self, $value ) = @_;
-    if ( ! defined($value) ) { $value = "" } # handle nulls
-    else {
-	$value =~ s/<.*?>//gs; # strip all html tags
-	$value =~ s/\s+/\ /g; # remove extra whitespace
-	$value =~ s/\&\w+\;//g; # remove &nbsp; etc
-    }
-    return $value;
-}
-
-# uses the given tag(s) to identify
-# the caption near to the image
-# (below, above or both below and above)
-sub extract_caption {
-    my ($self, $tag, $bound_tag, $contextref) = (@_);
-    my (@nt, $n, $etag, $gotcap);
-    return ("", "") if ( ! ($$contextref =~ /\Q$tag/) );
-
-    $nt[0] = $`;
-    $nt[1] = $';
-    $gotcap = 0;
-
-    # look before the image for a boundary tag
-    ($etag, $nt[0]) = $nt[0] =~ /<($bound_tag)[\s]?.*?>(.*?)$/is;
-    # if bound_tag too far from the image, then prob not caption
-    # (note: have to allow for tags, so multiply by 3
-    if ( $etag && length($nt[0]) < ($self->{'caption_length'} * 3) ) { 
-	if ( $nt[0] =~ /<\/$etag>/si ) {
-	    # the whole caption is above the image: <tag>text</tag><img>
-	    ($nt[0]) =~ /<(?:$etag)[\s]?.*?>(.*?)<\/$etag>/is;
-	    $nt[0] = $self->strip_tags($nt[0]);
-	    if ( length($nt[0]) > $self->{'min_near_text'} ) { 
-		$gotcap = 1;
-		$nt[1] = ""; 
-	    }
-
-	} elsif ( $nt[1] =~ /<\/$etag>/si) {
-	    # the caption tag covers image: <tag>text?<img>text?</tag>
-	    ($nt[1]) = $nt[1] =~ /(.*?)<\/$etag>/si;
-	    $nt[0] = $self->strip_tags($nt[0] . $nt[1]);
-	    if ( length($nt[0]) > $self->{'min_near_text'} ) { 
-		$gotcap = 2;
-		$nt[1] = "";
-	    }
-	}
-    }
-    # else try below the image
-    if ( ! $gotcap ) { 
-	# the caption is after the image: <img><tag>text</tag>
-	($etag, $nt[1]) = $nt[1] =~ /^.*?<($bound_tag)[\s]?.*?>(.*)/is;
-	if ( $etag && $nt[1] =~ /<\/$etag>/s) {
-	    ($nt[1]) = $nt[1] =~ /(.*?)<\/$etag>/si;
-	    $gotcap = 3;
-	    $nt[0] = "";
-	    $nt[1] = $self->strip_tags($nt[1]);
-	} 
-    }
-    if ( ! $gotcap ) { $nt[0] = $nt[1] = "" }
-    else {
-	# strip part-tags
-	$nt[0] =~ s/^.*>//s;
-	$nt[1] =~ s/<.*$//s;
-    }
-    my ($type);
-    if ( $gotcap == 0 ) { return ("nocaption", "") }
-    elsif ( $gotcap == 1 ) { $type = "captionabove:" }
-    elsif ( $gotcap == 2 ) { $type = "captioncovering:" }
-    elsif ( $gotcap == 3 ) { $type = "captionbelow:" }
-    return ($type, $nt[0], $nt[1]);
-}
-
-# tag-based near text
-# 
-# tries different tag sets
-# and chooses the best one
-sub extract_near_text {
-    my ($self, $tag, $textref, $prevpos, $pos, $nextpos) = (@_);
-    my (@neartext, $len, $hdelim, $maxtext, $mintext, $goodlen,
-	@bestlen, @best, $startpos, $context, $context_size,
-	$dist, $bdist, $best1, $i, $nt);
-    $bestlen[0] = $bestlen[1] = 0; $bestlen[2] = $bdist = 999999;
-    $best[0] = $best[1] = $best[2] = "";
-    $maxtext = $self->{'max_near_text'};
-    $mintext = $self->{'min_near_text'};
-    $goodlen = $self->{'neartext_length'}; 
-
-    # extract a context to extract near text from (faster)
-    $context_size = $maxtext*4;
-    $startpos = $pos - ($context_size / 2);
-    if ($startpos < $prevpos ) { $startpos = $prevpos }
-    if ($nextpos != -1 && $context_size > ( $nextpos - $startpos )) 
-    { $context_size = ($nextpos - $startpos) }
-    $context = substr ( $$textref, $startpos, $context_size );
-    $context =~ s/<!--.*?-->//gs;
-    $context =~ s/^.*-->(.*)/$1/gs;
-    $context =~ s/(.*)<!--.*$/$1/gs;
-
-    # try stepping through markup delimiter sets
-    # and selecting the best one
-    foreach $hdelim ( @{ $self->{'delims'} } ) {
-	@neartext = $self->extract_tagged_neartext($tag, $hdelim, \$context);
-	$nt = join("", @neartext);
-	$len = length($nt);
-	# Priorities:
-	# 1. Greater than mintext 
-	# 2. Less than maxtext 
-	# 3. Closest to goodlen
-	if ( $len <= $goodlen && $len > $bestlen[0] ) {
-	    $bestlen[0] = $len;
-	    $best[0] = $hdelim;
-	} elsif ( $len >= $maxtext && $len < $bestlen[2] ) {
-	    $bestlen[2] = $len;
-	    $best[2] = $hdelim;
-	} elsif ( $len >= $bestlen[0] && $len <= $bestlen[2] ) {
-	    $dist = abs($goodlen - $len);
-	    if ( $dist < $bdist ) {
-		$bestlen[1] = $len;
-		$best[1] = $hdelim;
-		$bdist = $dist;
-	    }
-	}
-    }
-    $best1 = 2;
-    foreach $i ( 0..2 ) {
-	if ( $bestlen[$i] == 999999 ) { $bestlen[$i] = 0 }
-	$dist = abs($goodlen - $bestlen[$i]);
-	if ( $bestlen[$i] > $mintext && $dist <= $bdist ) {
-	    $best1 = $i;
-	    $bdist = $dist;
-	}
-    }
-    @neartext = $self->extract_tagged_neartext($tag, $best[$best1], \$context);
-    if ( $bestlen[$best1] > $maxtext ) {
-	# truncate on word boundary if too much text
-	my $hmax = $maxtext / 2;
-	($neartext[0]) = $neartext[0] =~ /([^\s]*.{1,$hmax})$/s;
-	($neartext[1]) = $neartext[1] =~ /^(.{1,$hmax}[^\s]*)/s;
-    } elsif ( $bestlen[$best1] < $mintext ) {
-	# use plain text extraction if tags failed (e.g. usable tag outside context)
-	print {$self->{'outhandle'}} "HTMLImagePlugin: Fallback to plain-text extraction for $tag\n" 
-	    if $self->{'verbosity'} > 2;
-	$neartext[0] = "<tr><td>RawNeartext</td><td>" . $self->extract_raw_neartext($tag, $textref) . "</td></tr>";
-	$neartext[1] = "";
-    }
-    # get previous header if available
-    $neartext[0] .= "<br>\n" . 
-	$self->get_prev_header($pos, \$context) if ( $self->{'aggressiveness'} >= 4 );
-    $neartext[0] = " " if (! defined $neartext[0]);
-    $neartext[1] = " " if (! defined $neartext[1]);
-
-    return "<tr><td>NearText</td><td>" . (join "|",  @neartext) . "</td></tr>\n"; # TODO: the | is for testing purposes
-} # end extract_near_text
-
-# actually captures tag-based
-# near-text given a tag set
-sub extract_tagged_neartext {
-    my ($self, $tag, $bound_tag, $textref) = (@_);
-    return "" if ( ! ($$textref =~ /\Q$tag/) );
-    my (@nt, $delim, $pre_tag, $n);
-    $nt[0] = $`;
-    $nt[1] = $';
-
-    # get text after previous image tag
-    $nt[0] =~ s/.*<($bound_tag)[^>]*>(.*)/$2/is; # get rid of preceding text
-    if (defined($1)) { $delim = $1 }    
-    $pre_tag = $bound_tag;
-
-    if (defined($delim)) {
-	# we want to try and use the end tag of the previous delimiter
-	# (put it on the front of the list)
-	$pre_tag =~ s/(^|\|)($delim)($|\|)//i; # take it out
-	$pre_tag =~ s/\|\|/\|/i; # replace || with |
-	$pre_tag = $delim . "|" . $pre_tag; # put it on the front
-    }
-    
-    # get text before next image tag
-    $nt[1] =~ s/<\/?(?:$pre_tag)[^>]*>.*//is; # get rid of stuff after first delimiter
-
-    # process related text
-    for $n (0..1) {
-	if ( defined($nt[$n]) ) {
-	    $nt[$n] =~ s/<.*?>//gs; # strip all html tags
-	    $nt[$n] =~ s/\s+/\ /gs; # remove extra whitespace
-	    $nt[$n] =~ s/\&\w+\;//sg; # remove &nbsp; etc
-	    # strip part-tags
-	    if ( $n == 0 ) { $nt[0] =~ s/^.*>//s }
-	    if ( $n == 1 ) { $nt[1] =~ s/<.*$//s }
-	} else { $nt[$n] = ""; } # handle nulls
-    }
-    return @nt;
-}
-
-# this function is fall-back
-# if tags aren't suitable.
-#
-# extracts a fixed length of characters
-# either side of image tag (on word boundary)
-sub extract_raw_neartext {
-    my ($self, $tag, $textref) = (@_);
-    my ($rawtext, $startpos, $fp);
-    my $imgs = \%{$self->{'imglist'}};
-    ($fp) = $tag =~ /([\w\\\/]+\.(?:gif|jpe?g|png))/is;
-    if (! $fp) { return " " };
-    # if the cached, plain-text version isn't there, then create it
-    $self->init_plaintext($textref) unless defined($self->{'plaintext'});
-
-    # take the closest maxtext/2 characters 
-    # either side of the tag (by word boundary)
-    return "" if ( ! exists $imgs->{$fp}{'rawpos'} );
-    $startpos = $imgs->{$fp}{'rawpos'} - (($self->{'max_near_text'} / 2) + 20);
-    if ( $startpos < 0 ) { $startpos = 0 }
-    $rawtext = substr $self->{'plaintext'}, $startpos, $self->{'max_near_text'} + 20;
-    $rawtext =~ s/\s\s/ /g;
-
-    return $rawtext;
-}
-
-# init plaintext variable for HTML-stripped version 
-# (for full text index/raw assoc text extraction)
-sub init_plaintext {
-    my ($self, $textref) = (@_);
-    my ($page, $fp);
-    my $imgs = \%{$self->{'imglist'}};
-    $page = $$textref; # make a copy of original
-
-    # strip tags around image filenames so they don't get zapped
-    $page =~ s/<\w+\s+.*?([\w\/\\]+\.(?:gif|jpe?g|png))[^>]*>/\"$1\"/gsi;
-    $page =~ s/<.*?>//gs;
-    $page =~ s/&nbsp;/ /gs;
-    $page =~ s/&amp;/&/gs; #TODO: more &zzz; replacements (except &lt;, $gt;)
-
-    # get positions and strip images
-    while ( $page =~ /([^\s\'\"]+\.(jpe?g|gif|png))/ig ) {
-	$fp = $1;
-	if ( $imgs->{$fp}{'exists'} ) {
-	    $imgs->{$fp}{'rawpos'} = pos $page;
-	}
-	$page =~ s/\"$fp\"//gs;
-    }
-    $self->{'plaintext'} = $page;
-}
-
-# finds and filters images based on size 
-# (dimensions, height, filesize) and existence
-#
-# looks for image filenames (.jpg, .gif, etc)
-# and checks for existence on disk 
-# (hence supports most JavaScript images)
-sub get_img_list {
-    my $self = shift (@_);
-    my ($textref) = (@_);
-    my ($filepath, $relpath, $abspath, $pos, $num, $width, $height, $filesize);
-    my $imgs = \%{$self->{'imglist'}};
-    while ( $$textref =~ /([^\s\'\"]+\.(jpe?g|gif|png))/ig ) {
-	$filepath = $1;
-	$pos = pos $$textref;
-	next if ( $imgs->{$filepath}{'relpath'} );
-        $relpath = $filepath;
-	$relpath =~ s/^http\:\/\///; # remove http:// in case we have mirrored it
-	$relpath =~ s/\\/\//g;  # replace \ with /
-	$relpath =~ s/^\.\///s; # make "./filepath" into "filepath"
-	$imgs->{$filepath}{'relpath'} = $relpath;
-	$abspath = "$self->{'htpath'}/$relpath";
-
-	if (! -e $abspath) { next }
-
-	# can't modify real filepath var because it
-	# then can't be located in the page for tag recognition later
-	my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
-	($width, $height) = 
-	    `$imagick_cmd identify $abspath -ping -format "%wx%h"` =~ /^(\d*)x(\d*)$/m;
-	if (! ($width && $height)) { 
-	    print STDERR "HTMLImagePlugin: ($abspath) 'identify' failed. Check ImageMagicK binaries are installed and working correctly\n"; next;
-	}
-	$filesize = (-s $abspath);
-	if ( $filesize >= $self->{'min_size'} 
-	    && ( $width >= $self->{'min_width'} ) 
-	    && ( $height >= $self->{'min_height'} ) ) {
-	    
-	   $imgs->{$filepath}{'exists'} = 1;
-	   $imgs->{$filepath}{'pos'} = $pos;
-	   $imgs->{$filepath}{'width'} = $width;
-	   $imgs->{$filepath}{'height'} = $height;
-	   $imgs->{$filepath}{'filesize'} = $filesize;
-       } else {
-	   print {$self->{'outhandle'}} "HTMLImagePlugin: skipping $self->{'base_path'}/$relpath: $filesize, $width x $height\n" 
-	       if $self->{'verbosity'} > 2;
-       }
-    }
-    $num = 0;
-    foreach my $i ( keys %{$imgs} ) {
-	if ( $imgs->{$i}{'pos'} ) {
-	    $num++;
-	} else { delete $imgs->{$i} }
-    }
-    return $num;
-}
-
-# make the text available to the read function
-# by making it an object variable
-sub read_file {
-    my ($self, $filename, $encoding, $language, $textref) = @_;
-    $self->SUPER::read_file($filename, $encoding, $language, $textref);
-
-    # if HTMLplug has run through, then it will
-    # have replaced references so we have to 
-    # make a copy of the text before processing
-    if ( $self->{'index_pages'} ) {
-	$self->{'text'} = $$textref;
-	$self->{'textref'} = \($self->{'text'});
-    } else {
-	$self->{'textref'} = $textref;
-    }
-    $self->{'plaintext'} = undef;
-}
-
-# HTMLPlugin only extracts meta-data if it is specified in plugin options
-# hence a special function to do it here
-sub get_meta_value {
-    my ($self, $name, $textref) = @_;
-    my ($value);
-    $name = lc $name;
-    if ($name eq "title") {
-	($value) = $$textref =~ /<title>(.*?)<\/title>/is
-    } else {
-	my $qm = "(?:\"|\')";
-	($value) = $$textref =~ /<meta name\s*=\s*$qm?$name$qm?\s+content\s*=\s*$qm?(.*?)$qm?\s*>/is
-    }
-    $value = "" unless $value;
-    return $value;
-}
-
-# make filename an anchor reference
-# so we can go straight to the image
-# within the cached version of the source page
-# (augment's HTMLPlugin sub)
-sub replace_images {
-    my $self = shift (@_);
-    my ($front, $link, $back, $base_dir, 
-	$file, $doc_obj, $section) = @_;
-    $link =~ s/\"//g;
-    my ($a_name) = $link;
-    $a_name =~ s/[\/\\\:\&]/_/g;
-    # keep a list so we don't repeat the same anchor
-    if ( ! $self->{'imglist'}{$link}{'anchored'} ) {
-	$front = "<a name=\"gsdl_$a_name\">$front";
-	$back = "$back</a>";
-	$self->{'imglist'}{$link}{'anchored'} = 1;
-    }
-    return $self->SUPER::replace_images($front, $link, $back, $base_dir, 
-				     $file, $doc_obj, $section);
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ImageConverter.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ImageConverter.pm	(revision 26940)
+++ 	(revision )
@@ -1,527 +1,0 @@
-###########################################################################
-#
-# ImageConverter - helper plugin that does image conversion using ImageMagick
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package ImageConverter;
-
-use BaseMediaConverter;
-
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use util;
-use gsprintf 'gsprintf';
-
-BEGIN {
-    @ImageConverter::ISA = ('BaseMediaConverter');
-}
-
-# When used with multiple builder+buildproc, plugins loaded multiple times
-# => use this 'our' var to ensure only see the warning about ImageMagick once
-our $given_image_conversion_warning = 0;
-
-my $arguments = [
-      { 'name' => "create_thumbnail",
-	'desc' => "{ImageConverter.create_thumbnail}",
-	'type' => "enum",
-	'list' => [{'name' => "true", 'desc' => "{common.true}"},
-		   {'name' => "false", 'desc' => "{common.false}"}],
-	'deft' => "true",
-	'reqd' => "no" },
-      { 'name' => "thumbnailsize",
-	'desc' => "{ImageConverter.thumbnailsize}",
-	'type' => "int",
-	'deft' => "100",
-	'range' => "1,",
-	'reqd' => "no" },
-      { 'name' => "thumbnailtype",
-	'desc' => "{ImageConverter.thumbnailtype}",
-	'type' => "string",
-	'deft' => "gif",
-	'reqd' => "no" },
-      { 'name' => "noscaleup",
-	'desc' => "{ImageConverter.noscaleup}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "create_screenview",
-	'desc' => "{ImageConverter.create_screenview}",
-	'type' => "enum",
-	'list' => [{'name' => "true", 'desc' => "{common.true}"},
-		   {'name' => "false", 'desc' => "{common.false}"}],
-	'deft' => "true",
-	'reqd' => "no" },
-      { 'name' => "screenviewsize",
-	'desc' => "{ImageConverter.screenviewsize}",
-	'type' => "int",
-	'deft' => "500",
-	'range' => "1,",
-	'reqd' => "no" },
-      { 'name' => "screenviewtype",
-	'desc' => "{ImageConverter.screenviewtype}",
-	'type' => "string",
-	'deft' => "jpg",
-	'reqd' => "no" },
-      { 'name' => "converttotype",
-	'desc' => "{ImageConverter.converttotype}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "minimumsize",
-	'desc' => "{ImageConverter.minimumsize}",
-	'type' => "int",
-	'deft' => "100",
-	'range' => "1,",
-	'reqd' => "no" }
-		 ];
-
-my $options = { 'name' => "ImageConverter",
-		'desc' => "{ImageConverter.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args' => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BaseMediaConverter($pluginlist, $inputargs, $hashArgOptLists, 1);
-
-
-    return bless $self, $class;
-
-}
-
-# needs to be called after BasePlugin init, so that outhandle is set up.
-sub init {
-    my $self = shift(@_);
-
-    $self->{'tmp_file_paths'} = ();
-
-    # Check that ImageMagick is installed and available on the path 
-    my $image_conversion_available = 1;
-    my $no_image_conversion_reason = "";
-    # None of this works very well on Windows 95/98...
-    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
-	$image_conversion_available = 0;
-	$no_image_conversion_reason = "win95notsupported";
-    } else {
-	my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
-	my $result = `$imagick_cmd identify -help 2>&1`;
-	my $return_value = $?;
-
-	# When testing against non-zero return_value ($?), need to shift by 8 
-	# and convert it to its signed value. Linux returns -1 and Windows returns 
-	# 256 for "program not found". The signed equivalents are -1 and 1 respectively.
-	$return_value >>= 8;
-	$return_value = (($return_value & 0x80) ? -(0x100 - ($return_value & 0xFF)) : $return_value);
-
-	if ( ($ENV{'GSDLOS'} eq "windows" && $return_value == 1) || $return_value == -1) {  # Linux and Windows return different values for "program not found"
-	    $image_conversion_available = 0;
-	    $no_image_conversion_reason = "imagemagicknotinstalled";
-	}
-    }
-    $self->{'image_conversion_available'} = $image_conversion_available;
-    $self->{'no_image_conversion_reason'} = $no_image_conversion_reason;
-
-    if ($self->{'image_conversion_available'} == 0) {
-	if (!$given_image_conversion_warning) {
-	    my $outhandle = $self->{'outhandle'};
-	    &gsprintf($outhandle, "ImageConverter: {ImageConverter.noconversionavailable} ({ImageConverter.".$self->{'no_image_conversion_reason'}."})\n");
-	    $given_image_conversion_warning = 1;
-	}
-    }
-       
-}
-	
-
-# convert image to new type if converttotype is set
-# generate thumbnails if required
-# generate screenview if required
-# discover image metadata
-# filename_no_path must be in utf8 and url-encoded
-sub generate_images {
-    my $self = shift(@_);
-    my ($filename_full_path, $filename_encoded_full_path, $doc_obj, $section, $filename_encoding) = @_;
-
-    my ($unused_fefp,$filename_encoded_no_path)
-	= util::get_full_filenames("",$filename_encoded_full_path);
-
-    # The following is potentially very muddled thinking (but currently seems to work)
-    # generate_images currently called from ImagePlugin and PagedImagePlugin
-    my $filename_no_path = $filename_encoded_no_path; 
-
-    # check image magick status
-    return 0 if $self->{'image_conversion_available'} == 0;
-
-    # check the filenames
-    return 0 if ($filename_no_path eq "" || !-f $filename_full_path);
-
-    if ($self->{'enable_cache'}) {
-	$self->init_cache_for_file($filename_full_path);
-    }
-    if ($self->{'store_file_paths'}) {
-	$self->{'orig_file'} = "";
-	$self->{'thumb_file'} = "";
-	$self->{'screen_file'} = "";
-    }
-    my $verbosity = $self->{'verbosity'};
-    my $outhandle = $self->{'outhandle'};
-
-    # check the size of the image against minimum size if specified
-    my $minimumsize = $self->{'minimumsize'};
-    if (defined $minimumsize && (-s $filename_full_path < $minimumsize)) {
-        print $outhandle "ImageConverter: \"$filename_full_path\" too small, skipping\n"
-	    if ($verbosity > 1);
-	return 0; # or is there a better return value??
-    }
-    
-    my $filehead = $filename_no_path;
-    $filehead =~ s/\.([^\.]*)$//; # filename with no extension
-    my $assocfilemeta = "[assocfilepath]";
-    if ($section ne $doc_obj->get_top_section()) {
-	$assocfilemeta = "[parent(Top):assocfilepath]";
-    }
-
-    # The images that will get generated may contain percent signs in their src filenames
-    # Encode those percent signs themselves so that urls to the imgs refer to them correctly 
-    my $url_to_filehead = &unicode::filename_to_url($filehead);
-    my $url_to_filename_no_path = &unicode::filename_to_url($filename_no_path);
-    
-    # Convert the image to a new type (if required).
-    my $converttotype = $self->{'converttotype'};
-    my $type = "unknown";
-
-    if ($converttotype ne "" && $filename_full_path !~ m/$converttotype$/) {
-#	#    $doc_obj->add_utf8_metadata($section, "Image", $utf8_filename_meta);
-
-	my ($result, $converted_filename_full_path)
-	    = $self->convert($filename_full_path, $converttotype, "", "CONVERTTYPE");
-
-	$type = $converttotype;
-	$filename_full_path = $converted_filename_full_path;
-	$filename_no_path = "$filehead.$type";
-	$url_to_filename_no_path = "$url_to_filehead.$type";
-	if ($self->{'store_file_paths'}) {
-	    $self->{'orig_file'} = $converted_filename_full_path;
-	}
-    }
-
-    # add Image metadata 
-    $doc_obj->add_utf8_metadata($section, "Image", $url_to_filename_no_path); # url to generated image
-
-    # here we overwrite the original with the potentially converted one
-#    $doc_obj->set_utf8_metadata_element($section, "Source", &unicode::url_decode($filename_no_path)); # displayname of generated image
-#    $doc_obj->set_utf8_metadata_element($section, "SourceFile", $url_to_filename_no_path); # displayname of generated image
-
-#    $self->set_Source_metadata($doc_obj,$url_to_filename_no_path,undef);
-
-    my $raw_filename_full_path = &unicode::url_decode($filename_encoded_full_path);
-    $self->set_Source_metadata($doc_obj,$raw_filename_full_path,
-			       $filename_encoding, $section);
-
-
-    # use identify to get info about the (possibly converted) image
-    my ($image_type, $image_width, $image_height, $image_size) 
-	= &identify($filename_full_path, $outhandle, $verbosity);
-
-    if ($image_type ne " ") {
-	$type = $self->correct_mime_type($image_type);
-    }
-
-    #overwrite the ones added in BasePlugin
-    $doc_obj->set_metadata_element ($section, "FileFormat", $type);
-    $doc_obj->set_metadata_element ($section, "FileSize",   $image_size);
-
-    $doc_obj->add_metadata ($section, "ImageType",   $image_type);
-    $doc_obj->add_metadata ($section, "ImageWidth",  $image_width);
-    $doc_obj->add_metadata ($section, "ImageHeight", $image_height);
-    $doc_obj->add_metadata ($section, "ImageSize",   $image_size);
-
-    if ((defined $self->{'MaxImageWidth'}) 
-	&& ($image_width > $self->{'MaxImageWidth'})) {
-	$self->{'MaxImageWidth'} = $image_width;
-    }
-    if ((defined $self->{'MaxImageHeight'})
-	&& ($image_height > $self->{'MaxImageHeight'})) {
-	$self->{'MaxImageHeight'} = $image_height;
-    }
-
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($section, "srclink_file", $url_to_filename_no_path);
-    $doc_obj->add_metadata ($section, "srclinkFile", $url_to_filename_no_path);
-    $doc_obj->add_metadata ($section, "srcicon", "<img src=\"_httpprefix_/collect/[collection]/index/assoc/$assocfilemeta/[srclinkFile]\" width=\"[ImageWidth]\" height=\"[ImageHeight]\">");
-
-    # Add the image as an associated file
-    $doc_obj->associate_file($filename_full_path, $filename_no_path, "image/$type", $section);
-
-    if ($self->{'create_thumbnail'} eq "true") {
-	$self->create_thumbnail($filename_full_path, $filehead, $doc_obj, $section, $assocfilemeta, $url_to_filehead);
-    }
-    if ($self->{'create_screenview'} eq "true") {
-	$self->create_screenview($filename_full_path, $filehead, $doc_obj, $section, $assocfilemeta, $url_to_filehead, $image_width, $image_height);
-    }
-
-    return 1;
-}
-
-sub create_thumbnail {
-    my $self = shift(@_);
-    my ($original_file, $filehead, $doc_obj, $section, $assocfilemeta, $url_to_filehead) = @_;
-    $url_to_filehead = $filehead unless defined $url_to_filehead; 
-
-    my $thumbnailsize = $self->{'thumbnailsize'};
-    my $thumbnailtype = $self->correct_mime_type($self->{'thumbnailtype'});
-
-    # Generate the thumbnail with convert
-    my ($result,$thumbnailfile) 
-	= $self->convert($original_file, $thumbnailtype, "-geometry $thumbnailsize" . "x$thumbnailsize", "THUMB");
-    
-    # Add the thumbnail as an associated file ...
-    if (-e "$thumbnailfile") { 
-	$doc_obj->associate_file("$thumbnailfile", $filehead."_thumb.$thumbnailtype", 
-				 "image/$thumbnailtype",$section); # name of generated image
-	$doc_obj->add_metadata ($section, "ThumbType", $thumbnailtype);
-	$doc_obj->add_utf8_metadata ($section, "Thumb", $url_to_filehead."_thumb.$thumbnailtype"); # url to generated image
-	
-	$doc_obj->add_metadata ($section, "thumbicon", "<img src=\"_httpprefix_/collect/[collection]/index/assoc/$assocfilemeta/[Thumb]\" width=[ThumbWidth] height=[ThumbHeight]>");
-    
-	
-	# Extract Thumbnail metadata from convert output
-	if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
-	    $doc_obj->add_metadata ($section, "ThumbWidth", $1);
-	    $doc_obj->add_metadata ($section, "ThumbHeight", $2);
-	}
-	if ($self->{'store_file_paths'}) {
-	    $self->{'thumb_file'} = $thumbnailfile;
-	}
-
-    } else {
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "Couldn't find thumbnail $thumbnailfile\n";
-
-    }
-}
-
-sub create_screenview {
-    
-    my $self = shift(@_);
-    my ($original_file, $filehead, $doc_obj, $section, $assocfilemeta, $url_to_filehead, $image_width, $image_height) = @_;
-    $url_to_filehead = $filehead unless defined $url_to_filehead; 
-
-    my $screenviewsize = $self->{'screenviewsize'};
-    my $screenviewtype = $self->correct_mime_type($self->{'screenviewtype'});
-
-    # Scale the image, unless the original image is smaller than the screenview size and -noscaleup is set
-    my $scale_option = "-geometry $screenviewsize" . "x$screenviewsize";
-    if ($self->{'noscaleup'} && $image_width < $screenviewsize && $image_height < $screenviewsize)
-    {
-	$scale_option = "";
-    }
-    
-    # make the screenview image
-    my ($result,$screenviewfilename) 
-	= $self->convert($original_file, $screenviewtype, $scale_option, "SCREEN");    
-    
-    #add the screenview as an associated file ...
-    if (-e "$screenviewfilename") { 
-	$doc_obj->associate_file("$screenviewfilename", $filehead."_screen.$screenviewtype", "image/$screenviewtype",$section); # name of generated image
-	$doc_obj->add_metadata ($section, "ScreenType", $screenviewtype);
-	$doc_obj->add_utf8_metadata ($section, "Screen", $url_to_filehead."_screen.$screenviewtype"); # url to generated image
-	
-	$doc_obj->add_metadata ($section, "screenicon", "<img src=\"_httpprefix_/collect/[collection]/index/assoc/$assocfilemeta/[Screen]\" width=[ScreenWidth] height=[ScreenHeight]>");
-
-	# get screenview dimensions, size and type
-	if ($result =~ m/[0-9]+x[0-9]+=>([0-9]+)x([0-9]+)/) {
-	    $doc_obj->add_metadata ($section, "ScreenWidth", $1);
-	    $doc_obj->add_metadata ($section, "ScreenHeight", $2);
-	} elsif ($result =~ m/([0-9]+)x([0-9]+)/) {
-	    #if the image hasn't changed size, the previous regex doesn't match
-	    $doc_obj->add_metadata ($section, "ScreenWidth", $1);
-	    $doc_obj->add_metadata ($section, "ScreenHeight", $2);
-	}
-
-	if ($self->{'store_file_paths'}) {
-	    $self->{'screen_file'} = $screenviewfilename;
-	}
-
-    } else {
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "Couldn't find screenview file $screenviewfilename\n";
-
-    }
-
-}
-
-
-
-sub convert {
-    my $self = shift(@_);
-    my $source_file_path = shift(@_);
-    my $target_file_type = shift(@_);
-    my $convert_options  = shift(@_) || "";
-    my $convert_id       = shift(@_) || "";
-    my $cache_mode       = shift(@_) || "";
-
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-
-    my $source_file_no_path = &File::Basename::basename($source_file_path);
-
-    # Determine the full name and path of the output file
-    my $target_file_path;
-    if ($self->{'enable_cache'}) {
-	my $cached_image_dir = $self->{'cached_dir'};
-	my $image_root = $self->{'cached_file_root'};
-	$image_root .= "_$convert_id" if ($convert_id ne "");
-	my $image_file = "$image_root.$target_file_type";
-	$target_file_path = &util::filename_cat($cached_image_dir,$image_file);
-    }
-    else {
-	$target_file_path = &util::get_tmp_filename($target_file_type);
-	push(@{$self->{'tmp_file_paths'}}, $target_file_path);
-
-	# Output filename used to be parsed from result line:
-	#   my ($ofilename) = ($result =~ m/=>(.*\.$target_file_type)/);
-	# by the function that called 'convert'
-	# but this is no longer needed, as output filename is now
-	# explicitly passed back
-    }
-
-    # Generate and run the convert command
-    my $convert_command = "\"".&util::get_perl_exec()."\" -S gs-magick.pl convert -interlace plane -verbose $convert_options \"$source_file_path\" \"$target_file_path\"";
-
-    my $print_info = { 'message_prefix' => $convert_id,
-		       'message' => "Converting image $source_file_no_path to: $convert_id $target_file_type" };
-    $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");
-
-    my ($regenerated,$result,$had_error) 
-	= $self->autorun_general_cmd($convert_command,$source_file_path,$target_file_path,$print_info);
-
-    return ($result,$target_file_path);
-}
-
-sub convert_without_result {
-    my $self = shift(@_);
-
-    my $source_file_path = shift(@_);
-    my $target_file_type = shift(@_);
-    my $convert_options  = shift(@_) || "";
-    my $convert_id       = shift(@_) || "";
-
-    return $self->convert($source_file_path,$target_file_type,
-			  $convert_options,$convert_id,"without_result");
-}
-
-
-# Discover the characteristics of an image file with the ImageMagick
-# "identify" command.
-
-sub identify { 
-    my ($image, $outhandle, $verbosity) = @_;
-
-    # Use the ImageMagick "identify" command to get the file specs
-    my $command = "\"".&util::get_perl_exec()."\" -S gs-magick.pl identify \"$image\" 2>&1";
-    print $outhandle "$command\n" if ($verbosity > 2);
-    my $result = '';
-    $result = `$command`;
-    print $outhandle "$result\n" if ($verbosity > 3);
-
-    # Read the type, width, and height
-    my $type =   'unknown';
-    my $width =  'unknown';
-    my $height = 'unknown';
-
-    my $image_safe = quotemeta $image;
-    if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) {
-	$type = $1;
-	$width = $2;
-	$height = $3;
-    }
-
-    # Read the size
-    my $size = "unknown";
-    if ($result =~ m/^.* ([0-9]+)b/) {
-	$size = $1;
-    }
-    elsif ($result =~ m/^.* ([0-9]+)(\.([0-9]+))?kb?/) {
-	$size = 1024 * $1;
-	if (defined($2)) {
-	    $size = $size + (1024 * $2);
-	    # Truncate size (it isn't going to be very accurate anyway)
-	    $size = int($size);
-	}
-    }
-    elsif ($result =~ m/^.* ([0-9]+)(\.([0-9]+))?mb?/) {
-	$size = 1024 * 1024 * $1;
-        if (defined($2)) {
-	    $size = $size + (1024 * 1024 * $2);
-            # Truncate size (it isn't going to be very accurate anyway)
-            $size = int($size);
-        }
-     }
-    elsif ($result =~ m/^.* (([0-9]+)(\.([0-9]+))?e\+([0-9]+))(kb|b)?/) {
-	# Deals with file sizes on Linux of type "3.4e+02kb" where e+02 is 1*10^2.
-	# 3.4e+02 therefore evaluates to 3.4 x 1 x 10^2 = 340kb.
-	# Programming languages including Perl know how that 3.4e+02 is a number,
-	# so we don't need to do any calculations.
-	$size = $1*1; # turn the string into a number by multiplying it by 1
-	       #if we did $size = $1; $size would be merely the string "3.4e+02"
-	$size = int($size); # truncate size
-    }
-    print $outhandle "file: $image:\t $type, $width, $height, $size\n" 
-	if ($verbosity > 2);
-
-    # Return the specs
-    return ($type, $width, $height, $size);
-}
-
-sub clean_up_temporary_files {
-    my $self = shift(@_);
-
-    foreach my $tmp_file_path (@{$self->{'tmp_file_paths'}}) {
-	if (-e $tmp_file_path) {
-	    &util::rm($tmp_file_path);
-	}
-    }
-   
-}
-
-# image/jpg is not a valid mime-type, it ought to be image/jpeg. 
-# Sometimes JPEG is passed in also, want to keep things lowercase just in case.
-sub correct_mime_type {
-    my $self = shift(@_);
-    my ($file_extension) = @_;
-    
-    $file_extension = lc($file_extension);
-    $file_extension =~ s/jpg/jpeg/s; 
-
-    return $file_extension;
-}
-
-1;	
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/IndexPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/IndexPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,158 +1,0 @@
-###########################################################################
-#
-# IndexPlugin.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# This recursive plugin processes an index.txt file. 
-# The index.txt file should contain the list of files to be
-# included in the collection followed by any extra metadata to
-# be associated with each file.
-
-# The index.txt file should be formatted as follows:
-# The first line may be a key (beginning with key:)
-# to name the metadata fields
-# (e.g. key: Subject Organization Date) 
-# The following lines will contain a filename followed 
-# by the value that metadata entry is to be set to.
-# (e.g. 'irma/iw097e 3.2 unesco 1993' will associate the
-# metadata Subject=3.2, Organization=unesco, and Date=1993
-# with the file irma/iw097e if the above key line was used)
-
-# Note that if any of the metadata fields use the Hierarchy
-# classifier plugin then the value they're set to should
-# correspond to the first field (the descriptor) in the
-# appropriate classification file.
-
-# Metadata values may be named separately using a tag 
-# (e.g. <Subject>3.2) and this will override any name 
-# given to them by the key line.
-# If there's no key line any unnamed metadata value will be
-# named 'Subject'.
-
-package IndexPlugin;
-
-use plugin;
-use BasePlugin;
-use doc;
-use util;
-use cfgread;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @IndexPlugin::ISA = ('BasePlugin');
-}
-
-#my $arguments = [
-#		 ];
-
-my $options = { 'name'     => "IndexPlugin",
-		'desc'     => "{IndexPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes" };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    #push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-# return 1 if this class might recurse using $pluginfo
-sub is_recursive {
-    my $self = shift (@_);
-    
-    return 1;
-}
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $indexfile = &util::filename_cat($base_dir, $file, "index.txt");
-    if (!-f $indexfile) {
-	# not a directory containing an index file
-	return undef;
-    }
-
-    # found an index.txt file
-    print STDERR "<Processing n='$file' p='IndexPlugin'>\n" if ($gli);
-    print $outhandle "IndexPlugin: processing $indexfile\n";
-
-    # read in the index.txt
-    my $list = &cfgread::read_cfg_file ($indexfile, undef, '^[^#]\w');
-    my @fields = ();
-    # see if there's a 'key:' line
-    if (defined $list->{'key:'}) {
-	@fields = @{$list->{'key:'}};
-    }
-
-    my $index_base_dir = &util::filename_cat($base_dir, $file);
-
-    # process each document
-    my $count = 0;
-    foreach my $docfile (keys (%$list)) {
-	last if ($maxdocs != -1 && ($total_count + $count) >= $maxdocs);
-	$metadata = {}; # at present we can do this as metadata
-	                # will always be empty when it arrives
-	                # at this plugin - this might cause 
-	                # problems if things change though
-
-	# note that $list->{$docfile} is an array reference
-	if ($docfile !~ /key:/i) {
-	    my $i = 0;
-	    for ($i = 0; $i < scalar (@{$list->{$docfile}}); $i ++) {
-		if ($list->{$docfile}->[$i] =~ /^<([^>]+)>(.+)$/) {
-		    unless (defined ($metadata->{$1})) {
-			$metadata->{$1} = [];
-		    } 
-		    push (@{$metadata->{$1}}, $2);
-		} elsif (scalar @fields >= $i) {
-		    unless (defined ($metadata->{$fields[$i]})) {
-			$metadata->{$fields[$i]} = [];
-		    } 
-		    push (@{$metadata->{$fields[$i]}}, $list->{$docfile}->[$i]);
-		} else {
-		    $metadata->{'Subject'} = $list->{$docfile};
-		}
-	    }
-	    $count += &plugin::read ($pluginfo, $index_base_dir, $docfile, $block_hash, $metadata, $processor, $maxdocs, ($total_count +$count), $gli);
-	}
-    }
-
-    return $count; # was processed
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/OAIPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/OAIPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,558 +1,0 @@
-###########################################################################
-#
-# OAIPlug.pm -- basic Open Archives Initiative (OAI) plugin
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package OAIPlugin;
-
-use unicode;
-use util;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use ReadXMLFile;
-use ReadTextFile; # needed for subroutine textcat_get_language_encoding
-use metadatautil;
-use MetadataRead;
-
-# methods with identical signatures take precedence in the order given in the ISA list.
-sub BEGIN {
-    @OAIPlugin::ISA = ('MetadataRead', 'ReadXMLFile', 'ReadTextFile');
-}
-
-my $set_list =
-    [ { 'name' => "auto",
-	'desc' => "{OAIPlugin.metadata_set.auto}" },
-      { 'name' => "dc",
-	'desc' => "{OAIPlugin.metadata_set.dc}" } 
-      ];
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "metadata_set",
-	'desc' => "{OAIPlugin.metadata_set}",
-	'type' => "enumstring",
-	'reqd' => "no",
-	'list' => $set_list,
-	'deft' => "dc" },
-      { 'name' => "document_field",
-	'desc' => "{OAIPlugin.document_field}",
-	'type' => "metadata",
-	'reqd' => "no",
-	'deft' => "gi.Sourcedoc" }
-      ];
-
-my $options = { 'name'     => "OAIPlugin",
-		'desc'     => "{OAIPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists,1);
-    my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about modifying options
-	return bless $self, $class;
-    }
-    # trim any ex. from document field iff it's the only metadata namespace prefix    
-    $self->{'document_field'} =~ s/^ex\.([^.]+)$/$1/;
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)(\.oai)$^;
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "OAI-PMH";
-}
-
-sub xml_start_document {
-    my $self = shift (@_);
-    $self->{'in_metadata_node'} = 0;
-    $self->{'rawxml'} = "";
-    $self->{'saved_metadata'} = {};
-}
-
-sub xml_end_document {
-}
-
-sub xml_doctype {
-    my $self = shift(@_);
-
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    ##die "" if ($name !~ /^OAI-PMH$/);
-
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "OAIPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
-    print STDERR "<Processing n='$self->{'file'}' p='OAIPlugin'>\n" if $self->{'gli'};
-
-}
-
-
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat,$element) = @_;
-
-    my %attr_hash = %_;
-
-    my $attr = "";
-    map { $attr .= " $_=$attr_hash{$_}"; } keys %attr_hash;
-
-    $self->{'rawxml'} .= "<$element$attr>";
-
-    if ($element eq "metadata") {
-	$self->{'in_metadata_node'} = 1;
-	$self->{'metadata_xml'} = "";
-    }
-
-    if ($self->{'in_metadata_node'}) {
-	$self->{'metadata_xml'} .= "<$element$attr>";
-    }
-}
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    $self->{'rawxml'} .= "</$element>";
-
-    if ($self->{'in_metadata_node'}) {
-	$self->{'metadata_xml'} .= "</$element>";
-    }
-
-    if ($element eq "metadata") {
-	my $textref = \$self->{'metadata_xml'};
-	#my $metadata = $self->{'metadata'};
-	my $metadata = $self->{'saved_metadata'};
-	$self->extract_oai_metadata($textref,$metadata);
-
-	$self->{'in_metadata_node'} = 0;	
-    }
-
-
-}
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    $self->{'rawxml'} .= $_;
-
-    if ($self->{'in_metadata_node'}) {
-	$self->{'metadata_xml'} .= $_;
-    }
-}
-
-
-sub metadata_read {
-    my $self = shift (@_);  
-
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
-    
-    if (!$self->parse_file($filename_full_path, $file, $gli)) {
-	$self->{'saved_metadata'} = undef;
-	return undef;
-    }
-
-    my $new_metadata = $self->{'saved_metadata'};
-    $self->{'saved_metadata'} = undef;
-
-    # add the pretty metadata table as metadata
-    my $ppmd_table = $self->{'ppmd_table'};
-    $new_metadata->{'prettymd'} = $ppmd_table;
-    $self->{'ppmd_table'} = undef;
-      
-    my $document_metadata_field = $self->{'document_field'};
-    my $url_array = $new_metadata->{$document_metadata_field};
-    if (!defined $url_array) {
-	# try ex.
-	$url_array = $new_metadata->{"ex.$document_metadata_field"};
-    }
-    my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
-    ##print STDERR "$num_urls urls for $file\n";
-    my $srcdoc_exists = 0;
-    my $srcdoc_pos = 0;
-    my $filename_dir = &util::filename_head($filename_full_path);
-    
-    # filenames in extrametadata must be relative to current dir, as 
-    # DirectoryPlugin adds path info on itself
-    my ($filename_for_metadata) = $file =~ /([^\\\/]+)$/; # this assumes there will only be one record per oai file - is this always the case??
-    for (my $i=0; $i<$num_urls; $i++) {
-	
-	if ($url_array->[$i] !~ m/^(https?|ftp):/) {
-	    
-	    my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
-	    if (-e $src_filename) {
-		$srcdoc_pos = $i;
-		$srcdoc_exists = 1;
-		# get the slashes the right way, use filename_cat
-		$filename_for_metadata = &util::filename_cat($url_array->[$i]);
-		last;
-	    }
-	}
-    }
-    
-    if ($srcdoc_exists) {
-	$self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
-    }
-    else {
-	# save the rawxml for the source document
-	$self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 0;
-	$self->{'oai-files'}->{$file}->{'rawxml'} = $self->{'rawxml'};
-	$self->{'rawxml'} = "";
-    }
-    
-    # return all the metadata we have extracted to the caller.
-    # Directory plug will pass it back in at read time, so we don't need to extract it again.
-    # extrametadata keys should be regular expressions
-    $filename_for_metadata = &util::filename_to_regex($filename_for_metadata);
-    $extrametadata->{$filename_for_metadata} = $new_metadata;
-    push(@$extrametakeys, $filename_for_metadata);
-    if ($srcdoc_exists) {
-	if (!defined $extrametafile->{$filename_for_metadata}) {
-	    $extrametafile->{$filename_for_metadata} = {};
-	}
-	 #maps the file to full path
-	$extrametafile->{$filename_for_metadata}->{$file} = $filename_full_path;
-    }
-    return 1;
-    
-}
-
-
-sub read {
-    my $self = shift (@_);  
-    
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    if (!defined $self->{'oai-files'}->{$file}) {
-	return undef;
-    }
-        
-    my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
-    if ($srcdoc_exists) {
-	# do nothing more - all the metadata has been extracted and associated with the srcdoc
-	# no more need to access details of this $file => tidy up as you go
-	delete $self->{'oai-files'}->{$file};
-	return 0; # not processed here, but don't pass on to rest of plugins
-    }
-
-    my $filename = $file;
-    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
-
-    # Do encoding stuff on metadata
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
-
-    # create a new document
-    my $doc_obj = new doc ($filename, "indexed_doc", $self->{'file_rename_method'});
-    my $top_section = $doc_obj->get_top_section;
-    my $plugin_type = $self->{'plugin_type'};
-    
-    my ($filemeta) = $file =~ /([^\\\/]+)$/;
-    my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj, $filename, $filename_encoding);
-
-    $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-    $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
-    $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
-    $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
-    $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
-    
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    # this will include all the metadata from the oai file that we extracted
-    # during metadata_read
-    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
-    
-    # do plugin specific processing of doc_obj
-    my $text = $self->{'oai-files'}->{$file}->{'rawxml'};
-    delete $self->{'oai-files'}->{$file};
-
-    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1;
-    }
-    
-    # do any automatic metadata extraction
-    $self->auto_extract_metadata ($doc_obj);
-    
-    # add an OID
-    $self->add_OID($doc_obj);
-        
-    # process the document
-    $processor->process($doc_obj);
-    
-    $self->{'num_processed'} ++;
-    
-    return 1; # processed the file
-}
-
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    print STDERR "<Processing n='$file' p='OAIPlugin'>\n" if ($gli);
-    print $outhandle "OAIPlugin: processing $file\n"
-	if $self->{'verbosity'} > 1;
-
-    my $cursection = $doc_obj->get_top_section();
-
-##    $self->extract_metadata ($textref, $metadata, $doc_obj, $cursection);
-
-    # add text to document object
-
-#    $$textref =~ s/<(.*?)>/$1 /g;
-    $$textref =~ s/</&lt;/g;
-    $$textref =~ s/>/&gt;/g;
-    $$textref =~ s/\[/&#91;/g;
-    $$textref =~ s/\]/&#93;/g;
-
-    $doc_obj->add_utf8_text($cursection, $$textref);
-
-    return 1;
-}
-
-
-# Improvement is to merge this with newer version in MetadataPass
-
-sub open_prettyprint_metadata_table
-{
-    my $self = shift(@_);
-
-    my $att   = "width=100% cellspacing=2";
-    my $style = "style=\'border-bottom: 4px solid #000080\'";
-
-	$self->{'ppmd_table'} = "\n<table $att $style>";
-}
-
-sub add_prettyprint_metadata_line 
-{
-    my $self = shift(@_);
-    my ($metaname, $metavalue_utf8) = @_;
-
-    $metavalue_utf8 = &util::hyperlink_text($metavalue_utf8);
-
-    $self->{'ppmd_table'} .= "  <tr bgcolor=#b5d3cd>\n";
-    $self->{'ppmd_table'} .= "    <td width=30%>\n";
-    $self->{'ppmd_table'} .= "      $metaname\n";
-    $self->{'ppmd_table'} .= "    </td>\n";
-    $self->{'ppmd_table'} .= "    <td>\n";
-    $self->{'ppmd_table'} .= "      $metavalue_utf8\n";
-    $self->{'ppmd_table'} .= "    </td>\n";
-    $self->{'ppmd_table'} .= "  </tr>\n";
-
-}
-
-sub close_prettyprint_metadata_table
-{
-    my $self = shift(@_);
-
-    $self->{'ppmd_table'} .= "</table>\n";
-}
-
-my $qualified_dc_mapping = {
-    "alternative" => "dc.title",
-    "tableOfContents" => "dc.description",
-    "abstract" => "dc.description",
-    "created" => "dc.date",
-    "valid" => "dc.date",
-    "available" => "dc.date",
-    "issued" => "dc.date",
-    "modified" => "dc.date",
-    "dateAccepted" => "dc.date",
-    "dateCopyrighted" => "dc.date",
-    "dateSubmitted" => "dc.date",
-    "extent" => "dc.format",
-    "medium" => "dc.format",
-    "isVersionOf" => "dc.relation",
-    "hasVersion" => "dc.relation",
-    "isReplacedBy" => "dc.relation",
-    "replaces" => "dc.relation",
-    "isRequiredBy" => "dc.relation",
-    "requires" => "dc.relation",
-    "isPartOf" => "dc.relation",
-    "hasPart" => "dc.relation",
-    "isReferencedBy" => "dc.relation",
-    "references" => "dc.relation",
-    "isFormatOf" => "dc.relation",
-    "hasFormat" => "dc.relation",
-    "conformsTo" => "dc.relation",
-    "spatial" => "dc.coverage",
-    "temporal" => "dc.coverage",
-# these are now top level elements in our qualified dc metadata set
-#	"audience" => "dc.any",
-#	"accrualMethod" => "dc.any",
-#	"accrualPeriodicity" => "dc.any",
-#	"accrualPolicy" => "dc.any",
-#	"instructionalMethod" => "dc.any",
-#	"provenance" => "dc.any",
-#	"rightsHolder" => "dc.any",
-    "mediator" => "dc.audience",
-    "educationLevel" => "dc.audience",
-    "accessRights" => "dc.rights",
-    "license" => "dc.rights",
-    "bibliographicCitation" => "dc.identifier"
-    };
-
-sub remap_dc_metadata
-{
-    my $self = shift(@_);
-
-    my ($metaname) = @_;
-
-    my ($prefix,$name) = ($metaname =~ m/^(.*?)\.(.*?)$/);
-    
-    if (defined $qualified_dc_mapping->{$name}) {
-	
-	return $qualified_dc_mapping->{$name}."^".$name;
-    }
-    
-    
-    return $metaname; # didn't get a match, return param passed in unchanged
-}
-
-
-sub extract_oai_metadata {
-    my $self = shift (@_);
-    my ($textref, $metadata) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    $self->open_prettyprint_metadata_table();
-
-    if ($$textref =~ m/<metadata\s*>(.*?)<\/metadata\s*>/s)
-    {
-	my $metadata_text = $1;
-
-	# locate and remove outermost tag (ignoring any attribute information in top-level tag)
-	my ($outer_tagname,$inner_metadata_text) = ($metadata_text =~ m/<([^ >]+).*?>(.*?)<\/\1>/s);
-	# split tag into namespace and tag name
-	my($namespace,$top_level_prefix) = ($outer_tagname =~ m/^(.*?):(.*?)$/);
-	# sometimes, the dc namespace is not specified as the prefix in each element (like <dc:title>)
-	# but is rather defined in the wrapper element containing the various dc meta elements,
-	# like <dc><title></title><creator></creator></dc>.
-	# In such a case, we use this wrapper element as the top_level_prefix
-	
-	# if there was no prefix, then the tag itself becomes the top_level_prefix
-	if(!defined $top_level_prefix && defined $outer_tagname) {
-	    $top_level_prefix = $outer_tagname;
-	}
-
-	#process each element one by one
-	while ($inner_metadata_text =~ m/<([^ >]+).*?>(.*?)<\/\1>(.*)/s)
-	{
-
-	    my $metaname = $1;
-	    my $metavalue = $2;
-	    $inner_metadata_text = $3;
-
-	    # greenstone uses . for namespace, while oai uses :
-	    $metaname =~ s/:/\./;
-	    # if there is no namespace, then we use the outer tag name or 
-	    # namespace for this element
-	    if ($metaname !~ m/\./)
-	    {
-		$metaname = "$top_level_prefix.$metaname";
-	    }
-	    
-	    # if metadata set is auto, leave as is, otherwise convert to 
-	    # specified namespace
-	    if ($self->{'metadata_set'} ne "auto") {
-		if ($metaname !~ /^gi\./) { # hack to not overwrite gi metadata
-		    $metaname =~ s/^([^\.]*)\./$self->{'metadata_set'}\./;
-		    if ($self->{'metadata_set'} eq "dc") {
-			# convert qualified dc terms to gs version, e.g.
-			# spatial becomes coverage^spatial
-			$metaname = $self->remap_dc_metadata($metaname);
-		    }
-		}
-	    }
-
-	    # uppercase the first char of the name
-	    $metaname =~ s/\.(.)/\.\u$1/;
-	    $metavalue =~ s/\[/&#91;/g;
-	    $metavalue =~ s/\]/&#93;/g;
-
-	    # so that GLI can see this metadata, store here as ex.dc.Title etc
-	    my $ex_metaname = $metaname;
-	    $ex_metaname =~ s/^ex\.//; # remove any pre-existing ex. prefix
-	    $ex_metaname = "ex.$ex_metaname"; # at last can prefix ex.
-
-	    if (defined $metadata->{$ex_metaname})
-	    {
-		push(@{$metadata->{$ex_metaname}},$metavalue);
-
-	    }
-	    else
-	    {
-		$metadata->{$ex_metaname} = [ $metavalue ];
-	    }
-
-	    # but don't add ex to the pretty print line
-	    $self->add_prettyprint_metadata_line($metaname, $metavalue);
-	    
-	}
-    }
-
-    $self->close_prettyprint_metadata_table();
-}
-
-## we know from the file extension, so doesn't need to check the doctype
-sub check_doctype {
-
-    return 1;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/OggVorbisPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/OggVorbisPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,154 +1,0 @@
-###########################################################################
-#
-# OggVorbisPlug.pm -- A plugin for Ogg Vorbis audio files
-#
-# Original code by Christy Kuo
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 1999-2004 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package OggVorbisPlugin;
-
-
-use BasePlugin;
-use Ogg::Vorbis::Header::PurePerl;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-sub BEGIN {
-    @OggVorbisPlugin::ISA = ('BasePlugin');
-}
-
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "string",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "add_technical_metadata",
-	'desc' => "{OggVorbisPlugin.add_technical_metadata}",
-	'type' => "flag",
-	'deft' => "" }, 
-      { 'name' => "file_rename_method",
-	'desc' => "{BasePlugin.file_rename_method}",
-	'type' => "enum",
-	'deft' => &get_default_file_rename_method(), # by default rename imported files and assoc files using this encoding
-	'list' => $BasePlugin::file_rename_method_list,
-	'reqd' => "no"
-      } ];
-
-my $options = { 'name'     => "OggVorbisPlugin",
-		'desc'     => "{OggVorbisPlugin.desc}",
-		'inherits' => "yes",
-		'abstract' => "no",
-		'args'     => $arguments };
-
-
-# This plugin processes exported Ogg Vorbis files with the suffix ".ogg"
-sub get_default_process_exp
-{
-    return q^(?i)(\.ogg)$^;
-}
-
-# rename imported media files using base64 encoding by default
-# so that the urls generated will always work with external apps
-sub get_default_file_rename_method() {
-    return "base64";
-}
-
-sub new
-{
-    my ($class) = shift(@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-    
-    return bless $self, $class;
-}
-
-# we don't want to hash on the file
-sub get_oid_hash_type {
-    my $self = shift (@_);
-    return "hash_on_ga_xml";
-}
-
-sub process
-{
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    my $top_section = $doc_obj->get_top_section();
-    # Extract metadata
-    my $ogg = Ogg::Vorbis::Header::PurePerl->new($filename_full_path);
-
-    # Comments added to the file
-    foreach my $key ($ogg->comment_tags())
-    {
-	# Convert key to title case
-	my $keytc = uc(substr($key, 0, 1)) . substr($key, 1, length($key));
-	foreach my $value ($ogg->comment($key))
-	{
-	    if (defined $value && $value ne "") {
-		$doc_obj->add_metadata($top_section, $keytc, $value);
-	    }
-	}
-    }
-
-    # Technical data (optional)
-    if ($self->{'add_technical_metadata'}) {
-	foreach my $key (keys %{$ogg->info})
-	{
-	    # Convert key to title case
-	    my $keytc = uc(substr($key, 0, 1)) . substr($key, 1, length($key));
-	    my $value = $ogg->info->{$key};
-	    if (defined $value && $value ne "") {
-		$doc_obj->add_metadata($top_section, $keytc, $value);
-	    }
-	}
-    }
-
-    $doc_obj->add_metadata ($top_section, "FileFormat", "OggVorbis");
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($top_section, "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($top_section, "srclinkFile", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($top_section, "srcicon", "_iconogg_");
-
-    # add dummy text and NoText metadata which can be used to suppress the dummy text
-    $self->add_dummy_text($doc_obj, $top_section);
-
-    # Add the actual file as an associated file
-    my $assoc_file = $doc_obj->get_assocfile_from_sourcefile();
-    $doc_obj->associate_file($filename_full_path, $assoc_file, "VORBIS", $top_section);
-
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ReadTextFile.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ReadTextFile.pm	(revision 26940)
+++ 	(revision )
@@ -1,574 +1,0 @@
-###########################################################################
-#
-# ReadTxtFile.pm -- base class for import plugins that have plain text files
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999-2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ReadTextFile;
-
-use strict; 
-no strict 'subs';
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use Encode;
-
-use multiread;
-use encodings;
-use unicode;
-use textcat;
-use doc;
-use ghtml;
-use gsprintf 'gsprintf';
-
-use AutoExtractMetadata;
-
-sub BEGIN {
-    @ReadTextFile::ISA = ( 'AutoExtractMetadata' );
-}
-
-my $encoding_plus_auto_list = 
-    [ { 'name' => "auto",
-	'desc' => "{ReadTextFile.input_encoding.auto}" } ];
-push(@{$encoding_plus_auto_list},@{$BasePlugin::encoding_list});
-
-my $arguments =
-    [ { 'name' => "input_encoding",
-	'desc' => "{ReadTextFile.input_encoding}",
-	'type' => "enum",
-	'list' => $encoding_plus_auto_list,
-	'reqd' => "no" ,
-	'deft' => "auto" } ,
-      { 'name' => "default_encoding",
-	'desc' => "{ReadTextFile.default_encoding}",
-	'type' => "enum",
-	'list' => $BasePlugin::encoding_list,
-	'reqd' => "no",
-        'deft' => "utf8" },
-      { 'name' => "extract_language",
-	'desc' => "{ReadTextFile.extract_language}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "default_language",
-	'desc' => "{ReadTextFile.default_language}",
-	'type' => "string",
-	'deft' => "en",
-	'reqd' => "no" }
-      ];
-
-
-my $options = { 'name'     => "ReadTextFile",
-		'desc'     => "{ReadTextFile.desc}",
-		'abstract' => "yes",
-		'inherits' => "no",
-		'args'     => $arguments };
-
-
-
-sub new {
-    my $class = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists, $auxiliary) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new AutoExtractMetadata($pluginlist, $inputargs, $hashArgOptLists, $auxiliary);
-
-    return bless $self, $class;
-    
-}
-
-
-
-# The ReadTextFile read_into_doc_obj() function. This function does all the
-# right things to make general options work for a given plugin.  It reads in
-# a file and sets up a slew of metadata all saved in doc_obj, which
-# it then returns as part of a tuple (process_status,doc_obj)
-#
-# Much of this functionality used to reside in read, but it was broken
-# down into a supporting routine to make the code more flexible.  
-#
-# recursive plugins (e.g. RecPlug) and specialized plugins like those
-# capable of processing many documents within a single file (e.g.
-# GMLPlug) will normally want to implement their own version of
-# read_into_doc_obj()
-#
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read_into_doc_obj {
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    # should we move this to read? What about secondary plugins?
-    print STDERR "<Processing n='$file' p='$self->{'plugin_type'}'>\n" if ($gli);
-    my $pp_file = &util::prettyprint_file($base_dir,$file,$gli);
-    print $outhandle "$self->{'plugin_type'} processing $pp_file\n"
-	    if $self->{'verbosity'} > 1;
-
-    my ($filename_full_path, $filename_no_path) =  &util::get_full_filenames($base_dir, $file);
-
-    # Do encoding stuff
-    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
-    if ($self->{'verbosity'} > 2) {
-	print $outhandle "ReadTextFile: reading $file as ($content_encoding,$language)\n";
-    }
-
-    # create a new document
-    my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-    my $top_section = $doc_obj->get_top_section();
-
-    # this should look at the plugin option too...
-    $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
-
-    my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-
-    $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-    $doc_obj->add_utf8_metadata($top_section, "Encoding", $content_encoding);
-    
-    # read in file ($text will be in utf8)
-    my $text = "";
-    $self->read_file ($filename_full_path, $content_encoding, $language, \$text);
-
-    if (!length ($text)) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='File contains no text'>\n";
-	}
-	gsprintf($outhandle, "$self->{'plugin_type'}: {ReadTextFile.file_has_no_text}\n", $filename_full_path) if $self->{'verbosity'};
-
-	my $failhandle = $self->{'failhandle'};
-	gsprintf($failhandle, "$file: " . ref($self) . ": {ReadTextFile.empty_file}\n");
-	# print $failhandle "$file: " . ref($self) . ": file contains no text\n";
-	$self->{'num_not_processed'} ++;
-
-	return (0,undef); # what should we return here?? error but don't want to pass it on
-    }
-   
-    # do plugin specific processing of doc_obj
-    unless (defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
-	$text = '';
-	undef $text;
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return (-1,undef);
-    }
-    $text='';
-    undef $text;
-   
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->add_associated_files($doc_obj, $filename_full_path);
-    $self->extra_metadata ($doc_obj, $top_section, $metadata);
-
-     # do any automatic metadata extraction
-    $self->auto_extract_metadata ($doc_obj);
-
-
-    # if we haven't found any Title so far, assign one
-    $self->title_fallback($doc_obj,$top_section,$filename_no_path);
-
-    $self->add_OID($doc_obj);
-    
-    return (1,$doc_obj);
-}
-
-# uses the multiread package to read in the entire file pointed to
-# by filename and loads the resulting text into $$textref. Input text
-# may be in any of the encodings handled by multiread, output text
-# will be in utf8
-sub read_file {
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-
-    if (!-r $filename)
-    {
-	my $outhandle = $self->{'outhandle'};
-	gsprintf($outhandle, "{ReadTextFile.read_denied}\n", $filename) if $self->{'verbosity'};
-	# print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
-	return;
-    }
-    $$textref = "";
-    if (!open (FILE, $filename)) {
-	gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename);
-	die "\n";
-    }
-     
-    if ($encoding eq "ascii") {
-	# Replace file 'slurp' with faster implementation
-	sysread(FILE, $$textref, -s FILE);
-	
-	# The old slow way of reading in a file
-	#undef $/;
-	#$$textref = <FILE>;
-	#$/ = "\n";
-    } else {
-	my $reader = new multiread();
-	$reader->set_handle ('ReadTextFile::FILE');
-	$reader->set_encoding ($encoding);
-	$reader->read_file ($textref);
-    }
-
-    # At this point $$textref is a binary byte string
-    # => turn it into a Unicode aware string, so full
-    # Unicode aware pattern matching can be used.
-    # For instance: 's/\x{0101}//g' or '[[:upper:]]'
-    # 
-
-    $$textref = decode("utf8",$$textref);
-
-    close FILE;
-}
-
-
-# Not currently used
-sub UNUSED_read_file_usingPerlsEncodeModule {
-##sub read_file {
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-
-    if (!-r $filename)
-    {
-        my $outhandle = $self->{'outhandle'};
-        gsprintf($outhandle, "{ReadTextFile.read_denied}\n", $filename) if $self->{'verbosity'};
-        # print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
-        return;
-    }
-    $$textref = "";
-    if (!open (FILE, $filename)) {
-        gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_f
-or_reading} ($!)\n", $filename);
-        die "\n";
-    }
-
-    my $store_slash = $/;
-    undef $/;
-    my $text = <FILE>;
-    $/ = $store_slash;
-
-    $$textref = decode($encoding,$text);
-
-    close FILE;
-}
-
-
-sub read_file_no_decoding {
-    my $self = shift (@_);
-    my ($filename, $textref) = @_;
-
-    if (!-r $filename)
-    {
-	my $outhandle = $self->{'outhandle'};
-	gsprintf($outhandle, "{ReadTextFile.read_denied}\n", $filename) if $self->{'verbosity'};
-	# print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
-	return;
-    }
-    $$textref = "";
-    if (!open (FILE, $filename)) {
-	gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename);
-	die "\n";
-    }
-     
-    my $reader = new multiread();
-    $reader->set_handle ('ReadTextFile::FILE');
-    $reader->read_file_no_decoding ($textref);
-    
-    $self->{'reader'} = $reader;
-
-    close FILE;
-}
-
-
-sub decode_text {
-    my $self = shift (@_);
-    my ($raw_text, $encoding, $language, $textref) = @_;
-
-    my $reader = $self->{'reader'};
-    if (!defined $reader) {
-	gsprintf(STDERR, "ReadTextFile::decode_text needs to call ReadTextFile::read_file_no_decoding first\n");
-    }
-    else {
-	$reader->set_encoding($encoding);
-	$reader->decode_text($raw_text,$textref);
-
-	# At this point $$textref is a binary byte string
-	# => turn it into a Unicode aware string, so full
-	# Unicode aware pattern matching can be used.
-	# For instance: 's/\x{0101}//g' or '[[:upper:]]'	
-	
-	$$textref = decode("utf8",$$textref);
-    }
-}
-
-
-sub textcat_get_language_encoding {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my ($language, $encoding, $extracted_encoding);
-    if ($self->{'input_encoding'} eq "auto") {
-        # use textcat to automatically work out the input encoding and language
-        ($language, $encoding) = $self->get_language_encoding ($filename);
-    } elsif ($self->{'extract_language'}) {
-	# use textcat to get language metadata
-        ($language, $extracted_encoding) = $self->get_language_encoding ($filename);
-        $encoding = $self->{'input_encoding'};
-	# don't print this message for english... english in utf8 is identical
-	# to english in iso-8859-1 (except for some punctuation). We don't have
-	# a language model for en_utf8, so textcat always says iso-8859-1!
-        if ($extracted_encoding ne $encoding && $language ne "en" && $self->{'verbosity'}) {
-	    my $plugin_name = ref ($self);
-	    my $outhandle = $self->{'outhandle'};
-	    gsprintf($outhandle, "$plugin_name: {ReadTextFile.wrong_encoding}\n", $filename, $encoding, $extracted_encoding);
-        }
-    } else {
-        $language = $self->{'default_language'};
-        $encoding = $self->{'input_encoding'}; 
-    }
-    
-#    print STDERR "**** language encoding of contents of file $filename:\n\t****$language $encoding\n";
-
-    return ($language, $encoding);
-}
-
-
-# Uses textcat to work out the encoding and language of the text in 
-# $filename. All html tags are removed before processing.
-# returns an array containing "language" and "encoding"
-sub get_language_encoding {
-    my $self = shift (@_);
-    my ($filename) = @_;
-    my $outhandle = $self->{'outhandle'};
-    my $unicode_format = "";
-    my $best_language = "";
-    my $best_encoding = "";
-    
-
-    # read in file
-    if (!open (FILE, $filename)) {
-	gsprintf(STDERR, "ReadTextFile::get_language_encoding {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename);
-	# this is a pretty bad error, but try to continue anyway
-	return ($self->{'default_language'}, $self->{'input_encoding'});
-    }
-    undef $/;
-    my $text = <FILE>;
-    $/ = "\n";
-    close FILE;
-
-    # check if first few bytes have a Byte Order Marker
-    my $bom=substr($text,0,2); # check 16bit unicode
-    if ($bom eq "\xff\xfe") { # little endian 16bit unicode
-	$unicode_format="unicode";
-    } elsif ($bom eq "\xfe\xff") { # big endian 16bit unicode
-	$unicode_format="unicode";
-    } else {
-	$bom=substr($text,0,3); # check utf-8
-	if ($bom eq "\xef\xbb\xbf") { # utf-8 coded FEFF bom
-	    $unicode_format="utf8";
-#	} elsif ($bom eq "\xef\xbf\xbe") { # utf-8 coded FFFE bom. Error!?
-#	    $unicode_format="utf8";
-	}
-    }
-    
-    my $found_html_encoding = 0;
-    # handle html files specially
-    # XXX this doesn't match plugins derived from HTMLPlug (except ConvertTo)
-    if (ref($self) eq 'HTMLPlugin' ||
-	(exists $self->{'converted_to'} && $self->{'converted_to'} eq 'HTML')){
-
-	# remove comments in head, including multiline ones, so that we don't match on 
-	# inactive tags (those that are nested inside comments)
-	my ($head) = ($text =~ m/<head>(.*)<\/head>/si);
-	$head = "" unless defined $head; # some files are not proper HTML eg php files
-	$head =~ s/<!--.*?-->//sg;
-
-	# remove <title>stuff</title> -- as titles tend often to be in English
-	# for foreign language documents
-	$text =~ s!<title>.*?</title>!!si;
-
-	# see if this html file specifies its encoding
-	if ($text =~ /^<\?xml.*encoding="(.+?)"/) {
-	    $best_encoding = $1;
-	}
-	# check the meta http-equiv charset tag
-	elsif ($head =~ m/<meta http-equiv.*content-type.*charset=(.+?)\"/si) {		       
-	    $best_encoding = $1;
-	}
-	if ($best_encoding) { # we extracted an encoding
-	    $best_encoding =~ s/-+/_/g;
-	    $best_encoding = lc($best_encoding); # lowercase
-	    if ($best_encoding eq "utf_8") { $best_encoding = "utf8" }
-	    $found_html_encoding = 1;
-	    # We shouldn't be modifying this here!!
-	    #$self->{'input_encoding'} = $best_encoding;
-	}
-        
-	# remove all HTML tags
-	$text =~ s/<[^>]*>//sg;
-    }
-
-    # don't need to do textcat if we know the encoding now AND don't need to extract language
-    if($found_html_encoding && !$self->{'extract_language'}) { # encoding specified in html file
-	$best_language = $self->{'default_language'};
-    }
-
-    else { # need to use textcat to get either the language, or get both language and encoding
-	$self->{'textcat'} = new textcat() if (!defined($self->{'textcat'}));
-	
-	if($found_html_encoding) { # know encoding, find language by limiting search to known encoding
-	    my $results = $self->{'textcat'}->classify_contents_for_encoding(\$text, $filename, $best_encoding);
-	    
-	    my $language;
-	    ($language) = $results->[0] =~ m/^([^-]*)(?:-(?:.*))?$/ if (scalar @$results > 0);
-
-	    if (!defined $language || scalar @$results > 3) {
-		# if there were too many results even when restricting results by encoding,
-		# or if there were no results, use default language with the known encoding
-		$best_language = $self->use_default_language($filename);
-	    } 
-	    else { # fewer than 3 results means textcat is more certain, use the first result
-		$best_language = $language;		
-	    } 
-	}
-	else { # don't know encoding or language yet, therefore we use textcat
-	    my $results = $self->{'textcat'}->classify_contents(\$text, $filename);
-	    
-	    # if textcat returns 3 or less possibilities we'll use the first one in the list
-	    if (scalar @$results <= 3) { # results will be > 0 when we don't constrain textcat by an encoding
-		my ($language, $encoding) = $results->[0] =~ m/^([^-]*)(?:-(.*))?$/;
-
-		$language = $self->use_default_language($filename) unless defined $language;
-		$encoding = $self->use_default_encoding($filename) unless defined $encoding;
-
-		$best_language = $language;
-		$best_encoding = $encoding;
-	    }
-	    else { # if (scalar @$results > 3) {
-		if ($unicode_format) { # in case the first had a BOM
-		    $best_encoding=$unicode_format;
-		}
-		else {
-		    # Find the most frequent encoding in the textcat results returned
-		    # Returns "" if there's no encoding more frequent than another
-		    $best_encoding = $self->{'textcat'}->most_frequent_encoding($results);
-		}
-		
-		if ($best_encoding eq "") { # encoding still not set, use defaults
-		    $best_language = $self->use_default_language($filename);
-		    $best_encoding = $self->use_default_encoding($filename);
-		}
-		elsif (!$self->{'extract_language'}) { # know encoding but don't need to discover language
-		    $best_language = $self->use_default_language($filename);
-		}
-		else { # textcat again using the most frequent encoding or the $unicode_format set above
-		    $results = $self->{'textcat'}->classify_contents_for_encoding(\$text, $filename, $best_encoding);
-		    my $language;
-		    ($language) = $results->[0] =~ m/^([^-]*)(?:-(.*))?$/ if (scalar @$results > 0);
-		    if (!defined $language || scalar @$results > 3) { 
-			# if no result or too many results, use default language for the encoding previously found
-			$best_language = $self->use_default_language($filename);
-		    }
-		    else { # fewer than 3 results, use the language of the first result
-			$best_language = $language;
-		    }
-		}
-	    }
-	}
-    }
-
-    if($best_encoding eq "" || $best_language eq "") {
-	print STDERR "****Shouldn't happen: encoding and/or language still not set. Using defaults.\n";
-	$best_encoding = $self->use_default_encoding($filename) if $best_encoding eq "";
-	$best_language = $self->use_default_language($filename) if $best_language eq "";
-    }
-#    print STDERR "****Content language: $best_language; Encoding: $best_encoding.\n";
-
-
-    if ($best_encoding =~ /^iso_8859/ && &unicode::check_is_utf8($text)) {
-	# the text is valid utf8, so assume that's the real encoding
-	# (since textcat is based on probabilities)
-	$best_encoding = 'utf8';
-    }
-
-    # check for equivalents where textcat doesn't have some encodings...
-    # eg MS versions of standard encodings
-    if ($best_encoding =~ /^iso_8859_(\d+)/) {
-	my $iso = $1; # which variant of the iso standard?
-	# iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
-	if ($text =~ /[\x80-\x9f]/) {
-	    # Western Europe
-	    if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
-	    elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
-	    elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
-	    elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
-	    elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
-	    elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
-	    elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
-	}
-    }
-
-    if ($best_encoding !~ /^(ascii|utf8|unicode)$/ &&
-	!defined $encodings::encodings->{$best_encoding}) {
-	if ($self->{'verbosity'}) {
-	    gsprintf($outhandle, "ReadTextFile: {ReadTextFile.unsupported_encoding}\n",
-		     $filename, $best_encoding, $self->{'default_encoding'});
-	}
-	$best_encoding = $self->{'default_encoding'};
-    }
-
-    return ($best_language, $best_encoding);
-}
-
-
-sub use_default_language {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    if ($self->{'verbosity'}>2) {
-	gsprintf($self->{'outhandle'},
-		 "ReadTextFile: {ReadTextFile.could_not_extract_language}\n",
-		 $filename, $self->{'default_language'});
-    }
-    return $self->{'default_language'};
-}
-
-sub use_default_encoding {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    if ($self->{'verbosity'}>2) {
-	gsprintf($self->{'outhandle'},
-		 "ReadTextFile: {ReadTextFile.could_not_extract_encoding}\n",
-		 $filename, $self->{'default_encoding'});
-    }
-    return $self->{'default_encoding'};
-}
-
-# Overridden by exploding plugins (eg. ISISPlug)
-sub clean_up_after_exploding
-{
-    my $self = shift(@_);
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/StructuredHTMLPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/StructuredHTMLPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,417 +1,0 @@
-###########################################################################
-#
-# StructuredHTMLPlugin.pm -- html plugin with extra facilities for teasing out 
-# hierarchical structure (such as h1, h2, h3, or user-defined tags) in an 
-# HTML document
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-# This plugin is to process an HTML file where sections are divided by 
-# user-defined headings tags. As it is difficult to predict what user's definition
-# this plugin allows to detect the user-defined titles up to three levels (level1, level2, level3...)
-# as well as allows to get rid of user-defined Table of Content (TOC)...
-# format:e.g. level1 (Abstract_title|ChapterTitle|Referencing Heading) level2(SectionHeading)...
-
-package StructuredHTMLPlugin;
-
-use HTMLPlugin;
-use ImageConverter; # want the identify method
-use util;
-
-use strict; # every perl program should have this!
-no strict 'refs'; # make an exception so we can use variables as filehandles
-
-sub BEGIN {
-    @StructuredHTMLPlugin::ISA = ('HTMLPlugin');
-}
-
-my $arguments = 
-    [
-     { 'name' => "level1_header",
-       'desc' => "{StructuredHTMLPlugin.level1_header}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "" },
-     { 'name' => "level2_header",
-       'desc' => "{StructuredHTMLPlugin.level2_header}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "" },
-     { 'name' => "level3_header",
-       'desc' => "{StructuredHTMLPlugin.level3_header}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "" },
-     { 'name' => "title_header",
-       'desc' => "{StructuredHTMLPlugin.title_header}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "" },
-     { 'name' => "delete_toc",
-       'desc' => "{StructuredHTMLPlugin.delete_toc}",
-       'type' => "flag",
-       'reqd' => "no"},
-     { 'name' => "toc_header",
-       'desc' => "{StructuredHTMLPlugin.toc_header}",
-       'type' => "regexp",
-       'reqd' => "no",
-       'deft' => "" }     
-     ];
-
-my $options = { 'name'     => "StructuredHTMLPlugin",
-		'desc'     => "{StructuredHTMLPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $self = new HTMLPlugin($pluginlist, $inputargs, $hashArgOptLists);
-    
-    return bless $self, $class;
-}
-
-
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my @head_and_body = split(/<body/i,$$textref);
-    my $head = shift(@head_and_body);
-    my $body_text = join("<body", @head_and_body);
-    $head =~ m/<title>(.+)<\/title>/i;
-    my $doctitle = $1 if defined $1;
-    if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'}=~ /\S/) {
-	my @doc_properties = split(/<xml>/i,$head);
-	my $doc_heading = shift(@doc_properties);
-	my $rest_doc_properties = join(" ", @doc_properties);
-
-	my @extracted_metadata = split(/<\/xml>/i, $rest_doc_properties);
-	my $extracted_metadata = shift (@extracted_metadata);
-	$self->extract_metadata($extracted_metadata, $metadata, $doc_obj);
-    }
-    
-    # set the title here if we haven't found it yet
-    if (!defined $doc_obj->get_metadata_element ($doc_obj->get_top_section(), "Title")) {
-	if (defined $doctitle && $doctitle =~ /\S/) {
-	    $doc_obj->add_metadata($doc_obj->get_top_section(), "Title", $doctitle);
-	} else {
-	    $self->title_fallback($doc_obj,$doc_obj->get_top_section(),$file);
-	}
-    }
-
-    # If delete_toc is enabled, it means to get rid of toc and tof contents.
-    # get rid of TOC and TOF sections and their title
-    if (defined $self->{'delete_toc'} && ($self->{'delete_toc'} == 1)){
-	if (defined $self->{'toc_header'}&& $self->{'toc_header'} =~ /\S/){
-	    $body_text =~ s/<p class=(($self->{'toc_header'})[^>]*)>(.+?)<\/p>//isg;
-	}
-    }
-    
-    if (defined $self->{'title_header'} && $self->{'title_header'}=~ /\S/){
-	$self->{'title_header'} =~ s/^(\()(.*)(\))/$2/is;
-	$body_text =~ s/<p class=(($self->{'title_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h1>$3<\/h1><\/p>/isg;
-    }
-
-    if (defined $self->{'level1_header'} && $self->{'level1_header'}=~ /\S/ ){
-     	$self->{'level1_header'} =~ s/^\((.*)\)/$1/i;
-	$body_text =~ s/<p class=(($self->{'level1_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h1>$3<\/h1><\/p>/isg;
-    }
-    
-    if (defined $self->{'level2_header'} && $self->{'level2_header'}=~ /\S/){
-	$self->{'level2_header'} =~ s/^\((.*)\)/$1/i;
-	$body_text =~ s/<p class=(($self->{'level2_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h2>$3<\/h2><\/p>/isg;
-    }
-    
-    if (defined $self->{'level3_header'} && $self->{'level3_header'}=~ /\S/ ){
-	$self->{'level3_header'} =~ s/^\((.*)\)/$1/is;
-	$body_text =~ s/<p class=(($self->{'level3_header'})[^>]*)>(.+?)<\/p>/<p class=$1><h3>$3<\/h3><\/p>/isg;
-    }
-    
-    # Tidy up extra new lines
-    $body_text =~ s/(<p[^>]*><span[^>]*><o:p>&nbsp;<\/o:p><\/span><\/p>)//isg;
-    $body_text =~ s/(<p[^>]*><o:p>&nbsp;<\/o:p><\/p>)//isg;
-    
-    # what was the following line for. effectively unused. do we need it??
-    #$section_text .= "<!--\n<Section>\n-->\n";
-    #my $top_section_tag = "<!--\n<Section>\n-->\n";
-    #$body_text =~ s/(<div.*)/$top_section_text$doctitle$1/i;
-    #$body_text =~ s/(<div.*)/$top_section_tag$1/i;
-    my $body = "<body".$body_text;
-    
-    my $section_text = $head;
-    
-    # split HTML text on <h1>, <h2> etc tags
-    my @h_split = split(/<h/i,$body);
-    
-    my $hnum = 0;
-
-    my $sectionh1 = 0;
-    $section_text .= shift(@h_split);
-    
-    my $hc;
-    foreach $hc ( @h_split )
-    {
-	if ($hc =~ m/^([1-3])\s*.*?>(.*)$/s)
-	{
-	    my $new_hnum = $1;
-	    my $hc_after = $2;
-	    
-	    if ($hc_after =~ m/^(.*?)<\/h$new_hnum>/is)
-	    {
-		my $h_text = $1;
-		$hc =~ s/^(\&nbsp\;)+/\&nbsp\;/g;
-		# boil HTML down to some interesting text
-		$h_text =~ s/^[1-3]>//;
-		$h_text =~ s/<\/?.*?>//sg;
-		$h_text =~ s/\s+/ /sg;
-		$h_text =~ s/^\s$//s;
-		$h_text =~ s/(&nbsp;)+\W*/&nbsp;/sg;
-		
-		if ($h_text =~ m/\w+/)
-		{
-		    if ($new_hnum > $hnum)
-		    {
-			# increase section nesting
-			$hnum++;
-			while ($hnum < $new_hnum)
-			{
-			    my $spacing = "  " x $hnum;
-			    $section_text .= "<!--\n";
-			    $section_text .= $spacing."<Section>\n";
-			    $section_text .= "-->\n";
-			    $hnum++;
-			}
-		    }
-		    else # ($new_hnum <= $hnum)
-		    {
-			# descrease section nesting
-			while ($hnum >= $new_hnum)
-			{
-			    my $spacing = "  " x $hnum;
-			    $section_text .= "<!--\n";
-			    $section_text .= $spacing."</Section>\n";
-			    $section_text .= "-->\n";
-			    $hnum--;
-			}
-			$hnum++;
-		    }
-
-		    my $spacing = "  " x $hnum;
-		    $section_text .= "<!--\n";
-		    $section_text .= $spacing."<Section>\n";
-		    $section_text .= $spacing."  <Description>\n";
-		    $section_text .= $spacing."    <Metadata name=\"Title\">$h_text</Metadata>";
-		    $section_text .= $spacing."  </Description>\n";
-		    $section_text .= "-->\n";
-		    
-		    #print $outhandle $spacing."$h_text\n"
-		    #	if $self->{'verbosity'} > 2;
-		    
-		    $sectionh1++ if ($hnum==1);
-		}
-	    }
-	    else {
-###		print STDERR "***** hc = <h$hc\n\n";
-	    }
-	    $section_text .= "<h$hc";
-	}
-	else
-	{
-	    $section_text .= "<h$hc";
-	}
-    }
-
-    while ($hnum >= 1)
-    {
-	my $spacing = "  " x $hnum;
-	$section_text .= "<!--\n";
-	$section_text .= $spacing."</Section>\n";
-	$section_text .= "-->\n";
-	$hnum--;
-    }
-
-    $section_text .= "<!--\n</Section>\n-->\n";
-
-    $$textref = $section_text;
-    
-#    if ($sectionh1>0)
-#    {
-#	print $outhandle "  Located section headings ..."
-#	    if $self->{'verbosity'} > 1;
-#    }
-    
-    $$textref =~ s/<!\[if !vml\]>/<![if vml]>/g;
-    
-    $$textref =~ s/(&nbsp;)+/&nbsp;/sg;    
-    
-    ## $$textref =~ s/<o:p>&nbsp;<\/o:p>//g; # used with VML to space figures?
-    
-    $self->SUPER::process(@_);
-    
-}
-
-
-sub resize_if_necessary
-{
-    my ($self,$front,$back,$base_dir,$href) = @_;
-    
-    # dig out width and height of image, if there
-    my $img_attributes = "$front back";
-    my ($img_width)  = ($img_attributes =~ m/\s+width=\"?(\d+)\"?/i);
-    my ($img_height) = ($img_attributes =~ m/\s+height=\"?(\d+)\"?/i);
-    
-    # derive local filename for image based on its URL
-    my $img_filename = $href;
-    $img_filename =~ s/^[^:]*:\/\///;
-    $img_filename = &util::filename_cat($base_dir, $img_filename);
-    
-    # Replace %20's in URL with a space if required. Note that the filename
-    # may include the %20 in some situations
-    if ($img_filename =~ /\%20/) {
-	if (!-e $img_filename) {
-	    $img_filename =~ s/\%20/ /g;
-	}
-    }
-    if ((-e $img_filename) && (defined $img_width) && (defined $img_height)) {
-	# get image info on width and height
-	
-	my $outhandle = $self->{'outhandle'};
-	my $verbosity = $self->{'verbosity'};
-
-	my ($image_type, $actual_width, $actual_height, $image_size) 
-	    = &ImageConverter::identify($img_filename, $outhandle, $verbosity);
-	
-	#print STDERR "**** $actual_width x $actual_height";
-	#print STDERR " (requested: $img_width x $img_height)\n";
-
-	if (($img_width < $actual_width) || ($img_height < $actual_height)) {
-	    #print $outhandle "Resizing $img_filename\n" if ($verbosity > 0);
-	    
-	    # derive new image name based on current image
-	    my ($tailname, $dirname, $suffix)
-		= &File::Basename::fileparse($img_filename, "\\.[^\\.]+\$");
-	    
-	    my $resized_filename 
-		= &util::filename_cat($dirname, $tailname."_resized".$suffix);
-	    
-	    #print STDERR "**** suffix = $suffix\n";
-	    
-	    # Generate smaller image with convert
-	    my $newsize = "$img_width"."x$img_height";
-	    my $command = "convert -interlace plane -verbose "
-		."-geometry $newsize \"$img_filename\" \"$resized_filename\"";
-	    $command = "\"".&util::get_perl_exec()."\" -S gs-magick.pl $command";
-	    #print $outhandle "ImageResize: $command\n" if ($verbosity > 2);
-	    #my $result = '';
-	    #print $outhandle "ImageResize result: $result\n" if ($verbosity > 2);
-	}
-    }
-    return $href;
-}
-
-sub replace_images {
-    my $self = shift (@_);
-    my ($front, $link, $back, $base_dir, 
-	$file, $doc_obj, $section) = @_;
-    # remove quotes from link at start and end if necessary
-    if ($link=~/^\"/) {
-	$link=~s/^\"//;$link=~s/\"$//;
-	$front.='"';
-	$back="\"$back";
-    }
-    
-    $link =~ s/\n/ /g;
-    
-    my ($href, $hash_part, $rl) = $self->format_link ($link, $base_dir, $file);
-    
-##    $href = $self->resize_if_necessary($front,$back,$base_dir,$href);
-    
-    my $middle = $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section);
-    
-    return $front . $middle . $back;
-}
-
-sub extract_metadata 
-{
-    my $self = shift (@_);
-    my ($textref, $metadata, $doc_obj) = @_;
-    my $outhandle = $self->{'outhandle'};
-    
-    return if (!defined $textref);
-
-    my $separator = $self->{'metadata_field_separator'};
-    if ($separator eq "") {
-	undef $separator;
-    }
-    # metadata fields to extract/save. 'key' is the (lowercase) name of the
-    # html meta, 'value' is the metadata name for greenstone to use
-    my %find_fields = ();
-    my ($tag,$value);
-
-    my $orig_field = "";
-    foreach my $field (split /\s*,\s*/, $self->{'metadata_fields'}) {
-	# support tag<tagname>
-	if ($field =~ /^(.*?)\s*<(.*?)>$/) {
-	    # "$2" is the user's preferred gs metadata name
-	    $find_fields{lc($1)}=$2; # lc = lowercase
-	    $orig_field = $1;
-	} else { # no <tagname> for mapping
-	    # "$field" is the user's preferred gs metadata name
-	    $find_fields{lc($field)}=$field; # lc = lowercase
-	    $orig_field = $field;
-	}
-
-	if ($textref =~ m/<o:$orig_field>(.*)<\/o:$orig_field>/i){
-	    $tag = $orig_field;
-	    $value = $1;
-	    if (!defined $value || !defined $tag){
-		#print $outhandle "StructuredHTMLPlugin: can't find VALUE in \"$tag\"\n";
-		next;
-	    } else {
-		# clean up and add
-		chomp($value); # remove trailing \n, if any
-		$tag = $find_fields{lc($tag)};
-		#print $outhandle " extracted \"$tag\" metadata \"$value\"\n" 
-		#    if ($self->{'verbosity'} > 2);
-		if (defined $separator) {
-		    my @values = split($separator, $value);
-		    foreach my $v (@values) {
-			$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $v) if $v =~ /\S/;
-		    }
-		}
-		else {
-		    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), $tag, $value);
-		}
-	    }
-	}
-    }
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/UnknownPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/UnknownPlugin.pm	(revision 26940)
+++ 	(revision )
@@ -1,170 +1,0 @@
-###########################################################################
-#
-# UnknownPlugin.pm -- Plugin for files you know about but Greenstone doesn't
-#
-# A component of the Greenstone digital library software from the New
-# Zealand Digital Library Project at the University of Waikato, New
-# Zealand.
-#
-# Copyright (C) 2001 Gordon W. Paynter
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# UnknownPlugin - a plugin for unknown files
-
-# This is a simple Plugin for importing files in formats that
-# Greenstone doesn't know anything about.  A fictional document will
-# be created for every such file, and the file itself will be passed
-# to Greenstone as the "associated file" of the document.
-
-# Here's an example where it is useful: I have a collection of
-# pictures that include a couple of quicktime movie files with names
-# like DCP_0163.MOV.  Rather than write a new plugin for quicktime
-# movies, I add this line to the collection configuration file:
-
-# plugin UnknownPlugin -process_exp "*.MOV" -assoc_field "movie"
-
-# A document is created for each movie, with the associated movie
-# file's name in the "movie" metadata field.  In the collection's
-# format strings, I use the {If} macro to output different text for
-# each type of file, like this:
-
-# {If}{[movie],<HTML for displaying movie>}{If}{[Image],<HTML for displaying image>}
-
-# You can also add extra metadata, such as the Title, Subject, and
-# Duration, with metadata.xml files and RecPlug.  (If you want to use
-# UnknownPlugin with more than one type of file, you will have to add
-# some sort of distinguishing metadata in this way.)
-
-
-
-package UnknownPlugin;
-
-use BasePlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @UnknownPlugin::ISA = ('BasePlugin');
-}
-
-my $arguments =
-    [ { 'name' => "assoc_field",
-	'desc' => "{UnknownPlugin.assoc_field}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "file_format",
-	'desc' => "{UnknownPlugin.file_format}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "mime_type",
-	'desc' => "{UnknownPlugin.mime_type}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "srcicon",
-	'desc' => "{UnknownPlugin.srcicon}",
-	'type' => "string",
-	'deft' => "iconunknown",
-	'reqd' => "no" },
-      { 'name' => "process_extension",
-	'desc' => "{UnknownPlugin.process_extension}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "UnknownPlugin",
-		'desc'     => "{UnknownPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    # "-process_extension" is a simpler alternative to -process_exp for non-regexp people
-    if (!$self->{'process_exp'} && $self->{'process_extension'}) {
-	$self->{'process_exp'} = "\\." . $self->{'process_extension'} . "\$";
-    }
-
-    return bless $self, $class;
-}
-
-
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-
-    # check the filename is okay - do we need this??
-    if ($filename_full_path eq "" || $filename_no_path eq "") {
-	print $outhandle "UnknownPlugin: couldn't process \"$filename_no_path\"\n";
-	return undef;
-    }
-
-    # Add the file as an associated file ...
-    my $section = $doc_obj->get_top_section();
-    my $file_format = $self->{'file_format'} || "unknown";
-    my $mime_type = $self->{'mime_type'} || "unknown/unknown";
-    my $assoc_field = $self->{'assoc_field'} || "unknown_file";
-
-    # The assocfilename is the url-encoded version of the utf8 filename
-    my $assoc_file = $doc_obj->get_assocfile_from_sourcefile();
-
-    $doc_obj->associate_file($filename_full_path, $assoc_file, $mime_type, $section);
-    $doc_obj->add_metadata ($section, "FileFormat", $file_format);
-    $doc_obj->add_metadata ($section, "MimeType", $mime_type);
-    $doc_obj->add_utf8_metadata ($section, $assoc_field, $doc_obj->get_source()); # Source metadata is already in utf8 
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($section, "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($section, "srclinkFile", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($section, "srcicon", "_".$self->{'srcicon'}."_");
-    
-    # we have no text - add dummy text and NoText metadata
-    $self->add_dummy_text($doc_obj, $section);
-
-    return 1;
-}
-
-
-1;
-
-
-
-
-
-
-
-
-
-
-
