Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/BookPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/BookPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,246 +1,0 @@
-###########################################################################
-#
-# BookPlugin.pm (formally called HBSPlug) -- plugin for processing simple
-# html (or text) books
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# creates multi-level document from document containing 
-# <<TOC>> level tags. Metadata for each section is taken from any
-# other tags on the same line as the <<TOC>>. e.g. <<Title>>xxxx<</Title>>
-# sets Title metadata.
-
-# Everything else between TOC tags is treated as simple html (i.e. no 
-# processing of html links or any other HTMLPlug type stuff is done).
-
-# expects input files to have a .hb file extension by default (this can be 
-# changed by adding a -process_exp option
-
-# a file with the same name as the hb file but a .jpg extension is
-# taken as the cover image (jpg files are blocked by this plugin)
-
-# BookPlugin is a simplification (and extension) of the HBPlug used
-# by the Humanity Library collections. BookPlugin is faster as it expects
-# the input files to be cleaner (The input to the HDL collections
-# contains lots of excess html tags around <<TOC>> tags, uses <<I>>
-# tags to specify images, and simply takes all text between <<TOC>>
-# tags and start of text to be Title metadata). If you're marking up
-# documents to be displayed in the same way as the HDL collections,
-# use this plugin instead of HBPlug.
-
-package BookPlugin;
-
-use AutoExtractMetadata;
-use util;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @BookPlugin::ISA = ('AutoExtractMetadata');
-}
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_block_exp() } ];
-
-my $options = { 'name'     => "BookPlugin",
-		'desc'     => "{BookPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new AutoExtractMetadata($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub get_default_block_exp {
-    my $self = shift (@_);
-
-    return q^\.jpg$^;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.hb$^;
-}
-
-# do plugin specific processing of doc_obj
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    print STDERR "<Processing n='$file' p='BookPlugin'>\n" if ($gli);
-    print $outhandle "BookPlugin: processing $file\n" 
-	if $self->{'verbosity'} > 1;
-    
-    my $cursection = $doc_obj->get_top_section();
-
-    # Add FileFormat as the metadata
-    $doc_obj->add_metadata($doc_obj->get_top_section(),"FileFormat", "Book");
-
-    my $filename = &util::filename_cat($base_dir, $file);
-    my $absdir = $filename;
-    $absdir =~ s/[^\/\\]*$//;
-
-    # add the cover image
-    my $coverimage = $filename;
-    $coverimage =~ s/\.[^\.]*$/\.jpg/i;
-    $doc_obj->associate_file($coverimage, "cover.jpg", "image/jpeg");
-
-    my $title = "";
-
-    # remove any leading rubbish
-    $$textref =~ s/^.*?(<<TOC)/$1/ios;
-    
-    my $curtoclevel = 1;
-    my $firstsection = 1;
-    my $toccount = 0;
-    while ($$textref =~ /\w/) {
-	$$textref =~ s/^<<TOC(\d+)>>([^\n]*)\n(.*?)(<<TOC|\Z)/$4/ios;
-	my $toclevel = $1;
-	my $metadata = $2;
-	my $sectiontext = $3;
-
-	if ($toclevel == 2) {
-	    $toccount ++;
-	}
-	
-	# close any sections below the current level and
-	# create a new section (special case for the firstsection)
-	while (($curtoclevel > $toclevel) ||
-	       (!$firstsection && $curtoclevel == $toclevel)) {
-	    $cursection = $doc_obj->get_parent_section ($cursection);
-	    $curtoclevel--;
-	}
-	if ($curtoclevel+1 < $toclevel) {
-	    print $outhandle "WARNING - jump in toc levels in $filename " . 
-		"from $curtoclevel to $toclevel\n";
-	}
-	while ($curtoclevel < $toclevel) {
-	    $curtoclevel++;
-	    $cursection = 
-		$doc_obj->insert_section($doc_obj->get_end_child($cursection));
-	}
-
-	# sort out metadata
-	while ($metadata =~ s/^.*?<<([^>]*)>>(.*?)<<[^>]*>>//) {
-	    my $metakey = $1;
-	    my $metavalue = $2;
-
-	    if ($metavalue ne "" && $metakey ne "") {
-		# make sure key fits in with gsdl naming scheme 
-		$metakey =~ tr/[A-Z]/[a-z]/;
-		$metakey = ucfirst ($metakey);
-		$doc_obj->add_utf8_metadata ($cursection, $metakey, $metavalue);
-	    }
-	}
-
-	# remove header rubbish
-	$sectiontext =~ s/^.*?<body[^>]*>//ios;
-
-	# and any other unwanted tags
-	$sectiontext =~ s/<(\/p|\/html|\/body)>//isg;
-
-	# fix up the image links
-	$sectiontext =~ s/(<img[^>]*?src\s*=\s*\"?)([^\">]+)(\"?[^>]*>)/
-	    &replace_image_links($absdir, $doc_obj, $1, $2, $3)/isge;
-
-	# add the text
-	$doc_obj->add_utf8_text($cursection, $sectiontext);
-
-	$firstsection = 0;
-
-	$$textref =~ s/^\s+//s;
-    }
-
-    return 1;
-}
-
-sub replace_image_links {
-    my $self = shift (@_);
-    my ($dir, $doc_obj, $front, $link, $back) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my ($filename, $error);
-    my $foundimage = 0;
-    
-    $link =~ s/\/\///;
-    my ($imagetype) = $link =~ /([^\.]*)$/;
-    $imagetype =~ tr/[A-Z]/[a-z]/;
-    if ($imagetype eq "jpg") {$imagetype = "jpeg";}
-    if ($imagetype !~ /^(jpeg|gif|png)$/) {
-	print $outhandle "BookPlugin: Warning - unknown image type ($imagetype)\n";
-    }
-    my ($imagefile) = $link =~ /([^\/]*)$/;
-    my ($imagepath) = $link =~ /^[^\/]*(.*)$/;
-
-    if (defined $imagepath && $imagepath =~ /\w/) {
-	# relative link
-	$filename = &util::filename_cat ($dir, $imagepath);
-	if (-e $filename) {
-	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");
-	    $foundimage = 1;
-	} else {
-	    $error = "BookPlugin: Warning - couldn't find image file $imagefile in either $filename or";
-	}
-    }
-
-    if (!$foundimage) {
-	$filename = &util::filename_cat ($dir, $imagefile);
-	if (-e $filename) {
-	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");    
-	    $foundimage = 1;
-	} elsif (defined $error) {
-	    print $outhandle "$error $filename\n";
-	} else {
-	    print $outhandle "BookPlugin: Warning - couldn't find image file $imagefile in $filename\n";
-	}
-    }
-
-    if ($foundimage) {
-	return "${front}_httpdocimg_/${imagefile}${back}";
-    } else {
-	return "";
-    }
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/CSVPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/CSVPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,180 +1,0 @@
-###########################################################################
-#
-# CSVPlugin.pm -- A plugin for files in comma-separated value format
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 2006 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package CSVPlugin;
-
-
-use SplitTextFile;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-
-# CSVPlugin is a sub-class of SplitTextFile.
-sub BEGIN {
-    @CSVPlugin::ISA = ('SplitTextFile');
-}
-
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "split_exp",
-	'desc' => "{SplitTextFile.split_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_split_exp(),
-        'hiddengli' => "yes" }
-      ];
-
-
-my $options = { 'name'     => "CSVPlugin",
-		'desc'     => "{CSVPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-
-# This plugin processes files with the suffix ".csv"
-sub get_default_process_exp {
-    return q^(?i)(\.csv)$^;
-}
-
-    
-# This plugin splits the input text by line
-sub get_default_split_exp {
-    return q^\r?\n^;
-}
-
-
-sub new
-{
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}}, $options);
-
-    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-sub read_file
-{
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # Read the CSV file content
-    open(FILE, $filename);
-    my $reader = new multiread();
-    $reader->set_handle('CSVPlugin::FILE');
-    $reader->set_encoding($encoding);
-    $reader->read_file($textref);
-    close(FILE);
-
-    # Remove any blank lines so the data is split and processed properly
-    $$textref =~ s/\n(\s*)\n/\n/g;
-
-    # The first line contains the metadata element names
-    $$textref =~ s/^(.*?)\r?\n//;
-    my @csv_file_fields = ();
-    my $csv_file_field_line = $1 . ",";  # To make the regular expressions simpler
-    while ($csv_file_field_line ne "") {
-	# Handle quoted values
-	if ($csv_file_field_line =~ s/^\"(.*?)\"\,//) {
-	    my $csv_file_field = $1;
-	    $csv_file_field =~ s/ //g;  # Remove any spaces from the field names
-	    push(@csv_file_fields, $csv_file_field);
-	}
-	# Normal comma-separated case
-	elsif ($csv_file_field_line =~ s/^(.*?)\,//) {
-	    my $csv_file_field = $1;
-	    $csv_file_field =~ s/ //g;  # Remove any spaces from the field names
-	    push(@csv_file_fields, $csv_file_field);
-	}
-	# The line must be formatted incorrectly
-	else {
-	    print STDERR "Error: Badly formatted CSV field line: $csv_file_field_line.\n";
-	    last;
-	}
-    }
-    $self->{'csv_file_fields'} = \@csv_file_fields;
-}
-
-
-sub process
-{
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $section = $doc_obj->get_top_section();
-    my $csv_line = $$textref;
-    my @csv_file_fields = @{$self->{'csv_file_fields'}};
-
-    # Add the raw line as the document text
-    $doc_obj->add_utf8_text($section, $csv_line);
-
-    # Build a hash of metadata name to metadata value for this line
-    my $i = 0;
-    $csv_line .= ",";  # To make the regular expressions simpler
-    while ($csv_line ne "") {
-	# Metadata values containing commas are quoted
-	if ($csv_line =~ s/^\"(.*?)\"\,//) {
-	    # Only bother with non-empty values
-	    if ($1 ne "" && defined($csv_file_fields[$i])) {
-		$doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $1);
-	    }
-	}
-	# Normal comma-separated case
-	elsif ($csv_line =~ s/^(.*?)\,//) {
-	    # Only bother with non-empty values
-	    if ($1 ne "" && defined($csv_file_fields[$i])) {
-		$doc_obj->add_utf8_metadata($section, $csv_file_fields[$i], $1);
-	    }
-	}
-	# The line must be formatted incorrectly
-	else {
-	    print STDERR "Error: Badly formatted CSV line: $csv_line.\n";
-	    last;
-	}
-
-	$i++;
-    }
-
-    # Record was processed successfully
-    return 1;
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ConvertToRogPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ConvertToRogPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,457 +1,0 @@
-###########################################################################
-#
-# ConvertToRogPlugin.pm -- plugin that inherits from RogPlugin
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package ConvertToRogPlugin;
-
-use RogPlugin;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-use util;
-
-sub BEGIN {
-    @ConvertToRogPlugin::ISA = ('RogPlugin');
-}
-
-my $arguments = [
-		 ];
-my $options = { 'name'     => "ConvertToRogPlugin",
-		'desc'     => "{ConvertToRogPlugin.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes" };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new RogPlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    $self->{'convert_to'} = "Rog";
-    $self->{'convert_to_ext'} = "rog";
-
-    return bless $self, $class;
-}
-
-
-sub begin {
-    my $self = shift (@_);
-    
-    $self->SUPER::begin(@_);
-
-    $self->{'docnum'} = 0;
-}
-
-sub end {
-    my ($self) = @_;
-
-    # nothing to do, but keep symmetric with begin function
-    $self->SUPER::end(@_);
-}
-
-
-# Run conversion utility on the input file.  
-#
-# The conversion takes place in a collection specific 'tmp' directory so 
-# that we don't accidentally damage the input.
-#
-# The desired output type is indicated by $output_ext.  This is usually
-# something like "html" or "word", but can be "best" (or the empty string)
-# to indicate that the conversion utility should do the best it can.
-
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename, $textref) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $convert_to = $self->{'convert_to'};
-    my $failhandle = $self->{'failhandle'};
-
-    # softlink to collection tmp dir
-    my $tmp_dirname 
-	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
-    &util::mk_dir($tmp_dirname) if (!-e $tmp_dirname);
-
-    # derive tmp filename from input filename
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-
-    # Remove any white space from filename -- no risk of name collision, and
-    # makes later conversion by utils simpler. Leave spaces in path...
-    $tailname =~ s/\s+//g;
-    $tailname = $self->SUPER::filepath_to_utf8($tailname) unless &unicode::check_is_utf8($tailname);
-
-    my $tmp_filename = &util::filename_cat($tmp_dirname, "$tailname$suffix");
-
-    &util::soft_link($input_filename, $tmp_filename);
-
-    my $verbosity = $self->{'verbosity'};
-    if ($verbosity > 0) {
-	print $outhandle "Converting $tailname$suffix to $convert_to format\n";
-    }
-
-    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
-    
-    # Execute the conversion command and get the type of the result,
-    # making sure the converter gives us the appropriate output type
-    my $output_type = lc($convert_to);
-    my $cmd = "\"".&util::get_perl_exec()."\" -S gsMusicConvert.pl -verbose $verbosity -errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
-    $output_type = `$cmd`;
-
-    # remove symbolic link to original file
-    &util::rm($tmp_filename);
-
-    # Check STDERR here
-    chomp $output_type;
-    if ($output_type eq "fail") {
-	print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
-	print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
-	$self->{'num_not_processed'} ++;
-	if (-s "$errlog") {
-	    open(ERRLOG, "$errlog");
-	    while (<ERRLOG>) {
-		print $outhandle "$_";
-	    }
-	    print $outhandle "\n";
-	    close ERRLOG;
-	}
-	&util::rm("$errlog") if (-e "$errlog");
-	return "";
-    }
-
-    # store the *actual* output type and return the output filename
-    # it's possible we requested conversion to html, but only to text succeeded
-
-    $self->{'convert_to_ext'} = $output_type;
-    $self->{'converted_to'} = "Rog";
-
-    my $output_filename = $tmp_filename;
-
-    $output_filename =~ s/$suffix$//;
-
-    return $output_filename;
-}
-
-
-# Remove collection specific tmp directory and all its contents.
-
-sub cleanup_tmp_area {
-    my $self = shift (@_);
-
-    my $tmp_dirname 
-	= &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
-    &util::rm_r($tmp_dirname);
-    &util::mk_dir($tmp_dirname);
-}
-
-
-# Exact copy of read_rog_record from RogPlugin 
-# Needed for FILE in right scope
-
-sub read_rog_record
-{
-    my ($self,$file_buffer, $docnum, $seclevel) = @_;
-
-    my $next_line = $file_buffer->{'next_line'};
-
-    return 0 if (!defined $next_line);
-
-    if ($next_line eq "")
-    {
-	my $line;
-	while(defined($line=<FILE>))  
-	{
-	    $line =~ s/\r$//;
-	    $file_buffer->{'line_no'}++;
-	    next if ($line =~ m/^\#/);
-	    $next_line = $line;
-	    last;
-	}
-    }
-	    
-    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
-    {
-	print STDERR "Error: Malformed Rog file: $next_line";
-	return 0;
-    }
-    else
-    {
-	# init default values
-	$file_buffer->{'song'}->{'tempo'}    = 120;
-	$file_buffer->{'song'}->{'ks_type'}  = 0;
-	$file_buffer->{'song'}->{'ks_num'}   = 0;
-	$file_buffer->{'song'}->{'metadata'} = [];
-	$file_buffer->{'song'}->{'content'}  = "";
-	
-	$file_buffer->{'song'}->{'subcol'} = $2;
-	$file_buffer->{'song'}->{'title'}  = $4;
-	$file_buffer->{'song'}->{'tval'}   = $6;
-
-	chomp($next_line);
-	my $content = $next_line;
-	if (defined $docnum) 
-	{
-	    $content.= " $docnum $seclevel";
-	}
-	$content .= "\n";
-
-	$file_buffer->{'song'}->{'content'} = $content;
-
-
-	my $line;
-	while(defined($line=<FILE>))  
-	{
-	    $line =~ s/\r$//;
-
-	    $file_buffer->{'line_no'}++;
-	    next if ($line =~ m/^\#/);
-	
-	    if ($line =~ m/^song/) 
-	    {	
-		$file_buffer->{'next_line'} = $line;
-		return 1;
-	    }
-	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'tempo'} = $2;
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    } 
-	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'ks_type'} = $2;
-		$file_buffer->{'song'}->{'ks_num'}  = $4;
-		$file_buffer->{'song'}->{'content'} .= $line;	   
-	    } 
-	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'ts_numer'} = $2;
-		$file_buffer->{'song'}->{'ts_denom'} = $4;
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
-	    {
-		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	    else
-	    {
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	}
-	
-	$file_buffer->{'next_line'} = undef;
-    }
-
-    return 1;
-}
-
-# Override RogPlugin function so rog files are stored as sections (not docs)
-
-sub process_rog_record
-{
-    my ($self,$doc_obj,$cursection,$song) = @_;
-
-    $cursection = 
-	$doc_obj->insert_section($cursection);
-    $self->{'docnum'}++;
-
-    my $title = $song->{'title'};
-    my $title_safe = $title;
-    $title_safe =~ s/\'/\\\\&apos;/g;
-
-    # add metadata 
-    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
-    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
-    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
-    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
-    $doc_obj->add_metadata($cursection, "Title",         $title); 
-    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
-    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
-
-    foreach my $md ( @{$song->{'metadata'}} )
-    {
-	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
-    }
-
-    # add contents as text
-    $doc_obj->add_text($cursection,$song->{'content'});
-
-    return $cursection;
-}
-
-
-
-# Override BasePlugin read
-# We don't want to get language encoding stuff until after we've converted
-# our file to Rog format
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
- 
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-
-    # read in file ($text will be in utf8)
-    my $text = "";
-
-    my $output_ext = $self->{'convert_to_ext'};
-    my $conv_filename = $self->tmp_area_convert_file($output_ext, $filename_full_path);
-
-    if ("$conv_filename" eq "") {return 0;} # allows continue on errors
-    $self->{'conv_filename'} = $conv_filename;
-
-
-    # create a new document
-    #my $doc_obj = new doc ($conv_filename, "indexed_doc");
-    # the original filename is used now
-    my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-    # the converted filename is set separately
-    $doc_obj->set_converted_filename($conv_filename);
-
-    my $topsection = $doc_obj->get_top_section();
-    my $cursection = $topsection;
-
-    $self->{'docnum'}++;
-    my $docnum = $self->{'docnum'};
-
-    my ($filemeta) = $file =~ /([^\\\/]+)$/;
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj, $conv_filename, $filename_encoding);
-    
-    if ($self->{'cover_image'}) {
-	$self->associate_cover_image($doc_obj, $filename_full_path);
-    }
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $filename_full_path));
-
-    my $track_no = "1";
-    my $rog_filename = "$conv_filename$track_no.$output_ext";
-    while (1)
-    {
-	last unless open (FILE, $rog_filename) ;
-
-	my $file_buffer = { line_no => 0, next_line => "", song => {} };
-	
-	while ($self->read_rog_record($file_buffer, $docnum, $track_no))
-	{
-	    my $song = $file_buffer->{'song'};
-	    my $content = $song->{'content'};
-	    $content =~ s/^song\w+(.*)$/song $1 X.$track_no/;
-	    
-	    $cursection 
-		= $self->process_rog_record($doc_obj,$cursection,
-					    $file_buffer->{'song'});
-	}
-
-	close FILE;
-
-	$track_no++;
-	$rog_filename = "$conv_filename$track_no.$output_ext";
-    }
-
-    print STDERR "\n";
-
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
-    # do plugin specific processing of doc_obj
-    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1;
-    }
-    # do any automatic metadata extraction
-    $self->auto_extract_metadata ($doc_obj);
-    # add an OID
-    $self->add_OID($doc_obj);
-
-    my $oid = $doc_obj->get_OID();
-    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
-
-    $doc_obj->add_utf8_metadata ($topsection, "audiolink",  $appletlink); 
-    $doc_obj->add_utf8_metadata ($topsection, "audioicon",  "_iconaudio_"); 
-    $doc_obj->add_utf8_metadata ($topsection, "/audiolink", "</a>"); 
-
-    # if no title metadata defined, set it to filename minus extension
-    my $existing_title = $doc_obj->get_metadata_element($topsection,"Title");
-    if (!defined $existing_title) 
-    {
-	my $title = $doc_obj->get_metadata_element($topsection,"Source");
-	$title =~ s/\..*?$//g;
-	$doc_obj->add_utf8_metadata ($topsection, "Title", $title); 
-
-	my $title_safe = $title;
-	$title_safe =~ s/\'/\\\\&apos;/g;
-	$doc_obj->add_utf8_metadata ($topsection, "TitleSafe", $title_safe); 
-    }
-
-    # process the document
-    $processor->process($doc_obj);
-    $self->cleanup_tmp_area();
-
-    $self->{'num_processed'} ++;
-
-    return 1;
-}
-
-
-# do plugin specific processing of doc_obj for HTML type
-sub process_type {
-    my $self = shift (@_);
-    my ($doc_ext, $textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
-    
-    my $conv_filename = $self->{'conv_filename'};
-    my $tmp_dirname = File::Basename::dirname($conv_filename);
-    my $tmp_tailname = File::Basename::basename($conv_filename);
-    
-    my $converted_to = $self->{'converted_to'};
-    my $ret_val = 1;    
-
-#   $ret_val = &RogPlugin::process($self, $textref, $pluginfo,
-#				 $tmp_dirname, $tmp_tailname,
-#				 $metadata, $doc_obj);
-
-    # associate original file with doc object
-    my $cursection = $doc_obj->get_top_section();
-    my $filename = &util::filename_cat($base_dir, $file);
-    $doc_obj->associate_file($filename, "doc.$doc_ext", undef, $cursection);
-
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($cursection, "srclink_file", "doc.$doc_ext"); 
-    $doc_obj->add_metadata ($cursection, "srclinkFile", "doc.$doc_ext"); 
-    $doc_obj->add_utf8_metadata ($cursection, "srcicon",  "_icon".$doc_ext."_"); 
-
-    return $ret_val;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/DSpacePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DSpacePlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,427 +1,0 @@
-
-###########################################################################
-#
-# DSpacePlugin.pm -- plugin for importing a collection from DSpace 
-# 
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2004 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-# This plugin takes "contents" and dublin_core.xml file, which contain 
-# Metadata and lists of associated files for a particular document
-# and produces a document containing sections, one for each page.
-# The files should be named "contents" and "dublin_core.xml".  For each of 
-# document in DSpace, it is stored in one directory
-#
-# The format of the "contents" file is as follows:
-# 
-# File.type      bundle:ORIGINAL
-# license.txt    bundle:LICENSE
-# The format of the "dublin_core.xml" file is as follows:
-# The first line contains any metadata for the whole document
-# <dublin_core>
-# eg.
-# <dcvalue element="Title" qualifier="">Snail farming</dcvalue>
-# <dcvalue element="date" qualifier="">2004-10-15</dcvalue>
-#
-
-package DSpacePlugin;
-
-use ReadTextFile;
-use plugin;
-use XMLParser;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @DSpacePlugin::ISA = ('ReadTextFile');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "string",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "only_first_doc",
-	'desc' => "{DSpacePlugin.only_first_doc}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "first_inorder_ext",
-	'desc' => "{DSpacePlugin.first_inorder_ext}",
-	'type' => "string",
-	'reqd' => "no" },
-      { 'name' => "first_inorder_mime",
-	'desc' => "{DSpacePlugin.first_inorder_mime}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_block_exp(),
-	'reqd' => "no" }];
-
-
-my $options = { 'name'     => "DSpacePlugin",
-		'desc'     => "{DSpacePlugin.desc}",
-		'inherits' => "yes",
-		'abstract' => "no",
-		'args'     => $arguments };
-
-
-my $primary_doc_lookup = { 'text/html' => '(?i)\.(gif|jpe?g|jpe|jpg|png|css)$' };
-
-# Important variation to regular plugin structure.  Need to declare
-# $self as global variable to file so XMLParser callback routines
-# can access the content of the object. 
-my ($self); 
-
-sub new {
-    my ($class) = shift (@_); 
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
-    
-    if ($self->{'info_only'}) {
-	# don't worry about creating the XML parser as all we want is the 
-	# list of plugin options
-	return bless $self, $class;
-    }
-
-    #create XML::Parser object for parsing dublin_core.xml files
-    my $parser = new XML::Parser('Style' => 'Stream',
-				 'Handlers' => {'Char' => \&Char,
-						'Doctype' => \&Doctype
-						});
-    $self->{'parser'} = $parser;
-    $self->{'extra_blocks'} = {};
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)(contents)$^;
-}
-
-# want to block all files except the "contents"
-sub get_default_block_exp {
-    my $self = shift (@_);
-    
-    # Block handle and txt files if present. Specifically don't block dublin_core xml
-    return q^(?i)(handle|\.tx?t)$^;
-}
-
-sub store_block_files_BACKUP
-{
-    # Option of making blocking sensitive to files that are in directory
-    # This subroutine is not currently used! (relies on default block expression stopping all handle and .txt files)
-
-    my $self =shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-
-    my ($tailname, $contents_basedir, $suffix) = &File::Basename::fileparse($filename_full_path, "\\.[^\\.]+\$");
-    my $handle_filename = &util::filename_cat($contents_basedir,"handle");
-
-    if (&util::fd_exists($handle_filename)) {
-	&util::block_filename($block_hash,$handle_filename);
-    }
-}
-
-sub read_content
-{
-    my $self = shift (@_);
-    my ($dir, $only_first_doc, $first_inorder_ext, $first_inorder_mime, $mimetype_list) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my @fnamemime_list = ();
-    my @assocmime_list = ();
-
-    my $content_fname = &util::filename_cat($dir,"contents");
-
-    open(CIN,"<$content_fname") 
-	|| die "Unable to open $content_fname: $!\n";
-    
-    my $line;
-    my $pos = 0;
-
-    while (defined ($line = <CIN>)) {
-	if ($line =~ m/^(.*)\s+bundle:ORIGINAL\s*$/) {
-	    my $fname = $1;
-	    my $mtype = $mimetype_list->[$pos];
-	    my $fm_rec = { 'file' => $fname, 'mimetype' => $mtype};
-	    push(@fnamemime_list,$fm_rec);
-	    $pos++;
-	}
-    }
-    close CIN;
-
-    if ($only_first_doc){
-	my ($first_fname, @rest_fnames) = @fnamemime_list;
-	@fnamemime_list = ($first_fname);
-	@assocmime_list = @rest_fnames;
-    }
-
-    # allow user to specify the types of files (inorder)they would like to assign as
-    # a primary bitstream
-    if ($first_inorder_ext) {
-	# parse user-define file extension names
-	my @extfiles_list = split /,/, $first_inorder_ext;
-	my (@rest_fnames) = @fnamemime_list;
-	my @matched_list = ();
-	foreach my $file_ext (@extfiles_list) {
-	    $pos = 0;
-	    foreach my $allfiles (@fnamemime_list){
-		$allfiles->{'file'} =~ /^(.*)\.(.*?)$/;
-		my $allfiles_ext = $2;
-
-		if ($allfiles_ext =~ /$file_ext/) {
-		    print $outhandle "Existing file:$allfiles->{'file'} match the user-define File Extension:$file_ext\n";
-		    push (@matched_list, $allfiles);
-
-		    # delete the matched extension file from the array
-		    splice(@rest_fnames,$pos,1);
-
-		    return (\@matched_list, \@rest_fnames);
-
-		}
-		$pos++;
-	    }
-	}
-    }
-    
-    if ($first_inorder_mime) {
-	# parse user-define file mimetype
-	my @file_mime_list = split /,/, $first_inorder_mime;
-	my (@rest_fnames) = @fnamemime_list;
-	my @matched_list = ();
-	foreach my $file_mime (@file_mime_list) {
-	    $pos = 0;
-	    foreach my $allfiles (@fnamemime_list){
-		my $allfiles_mime = $allfiles->{'mimetype'};
-
-		if ($allfiles_mime =~ /$file_mime/) {
-		    print $outhandle "Existing file:$allfiles->{'file'} match the user-defined File MimeType:$file_mime\n";
-		    push (@matched_list, $allfiles);
-
-		    # delete the matched MIMEType file from the array
-		    splice(@rest_fnames,$pos,1);
-		    return (\@matched_list, \@rest_fnames);
-		}
-		$pos++;
-	    }
-	}
-    }
-    return (\@fnamemime_list, \@assocmime_list);
-}
-
-
-sub filemime_list_to_re
-{
-    my $self = shift (@_);
-    my ($fnamemime_list) = @_;
-
-    my @fname_list = map { "(".$_->{'file'}.")" } @$fnamemime_list;
-    my $fname_re = join("|",@fname_list);
-    $fname_re =~ s/\./\\\./g;
-
-    return $fname_re;
-}
-
-# Read dublin_core metadata from DSpace collection 
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    my $only_first_doc = $self->{'only_first_doc'};
-    my $first_inorder_ext = $self->{'first_inorder_ext'};
-    my $first_inorder_mime = $self->{'first_inorder_mime'};
-    
-    my $outhandle = $self->{'outhandle'};
-    
-    my $filename = &util::filename_cat($base_dir, $file);
-    # return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
-    
-    if ($filename !~ /dublin_core\.xml$/ || !-f $filename) {
-	return undef;
-    }
-    
-    print $outhandle "DSpacePlugin: extracting metadata from $file\n"
-	if $self->{'verbosity'} > 1;
-    
-    my ($dir) = $filename =~ /^(.*?)[^\/\\]*$/;
-
-    eval {
-	$self->{'parser'}->parsefile($filename);
-    };
-    
-    if ($@) {
-	die "DSpacePlugin: ERROR $filename is not a well formed dublin_core.xml file ($@)\n";
-    }
-
-    my $mimetype_list = $self->{'saved_metadata'}->{'ex.dc.Format^mimetype'};
-    my ($doc_file_mimes, $assoc_file_mimes) = $self->read_content($dir, $only_first_doc, $first_inorder_ext, 
-								  $first_inorder_mime, $mimetype_list);
-
-    my $file_re = $self->filemime_list_to_re($doc_file_mimes);
-
-    if ($only_first_doc || $first_inorder_ext || $first_inorder_mime) {	
-	foreach my $afm ( @$assoc_file_mimes ) {
- 	    my $full_af = &util::filename_cat($dir,$afm->{'file'});
-	    $self->{'extra_blocks'}->{$full_af} = 1;
-	}
-    }
-    push(@$extrametakeys,$file_re);
-
-    # See Format's Extent section in http://dublincore.org/documents/usageguide/qualifiers.shtml
-    # it could specify duration, size or even dimensions of the resource. It may be a useful piece
-    # of metadata to preserve after all.
-    #if (defined $self->{'saved_metadata'}->{'ex.dc.Format^extent'}) {
-	#delete $self->{'saved_metadata'}->{'ex.dc.Format^extent'};
-    #}
-
-    if (defined $mimetype_list) {
-	delete $self->{'saved_metadata'}->{'ex.dc.Format^mimetype'};
-
-	# Temporarily store associate file info in metadata table
-	# This will be removed in 'extra_metadata' in BasePlugin and used
-	# to perform the actual file association (once the doc obj has
-	# been formed
-
-	my $main_doc = $doc_file_mimes->[0];
-	my $md_mimetype = $main_doc->{'mimetype'};
-
-	my $pd_lookup = $primary_doc_lookup->{$md_mimetype};
-
-	if (defined $pd_lookup) {
-	    my $filter_re = $pd_lookup;
-	    @$assoc_file_mimes = grep { $_->{'file'} !~ m/$filter_re/ }  @$assoc_file_mimes;
-	}
-
-	my @gsdlassocfile_tobe 
-	    = map { &util::filename_cat($dir,$_->{'file'}) .":".$_->{'mimetype'}.":" } @$assoc_file_mimes if @$assoc_file_mimes;
-	$self->{'saved_metadata'}->{'gsdlassocfile_tobe'} = \@gsdlassocfile_tobe;
-
-    }
-    
-    $extrametadata->{$file_re} = $self->{'saved_metadata'};
-
-    return 1;
-}
-
-
-# The DSpacePlugin read() function. We are not actually reading any documents 
-# here, just blocking ones that have been processed by metadata read.
-#
-# Returns 0 for a file its blocking, undef for any other
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-    
-    # Block all files except contents
-    my $filename = &util::filename_cat($base_dir, $file);
-    return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
-
-    my $assocfile = $metadata->{'assocfile'};
-
-    return 0 if (($filename =~ /dublin_core\.xml$/) || ($filename =~ /contents$/));
-    return 0 if (defined $self->{'extra_blocks'}->{$filename});
-    return undef;
-}
-
-sub Doctype {
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    die if ($name !~ /^dublin_core$/);
-}
-
-sub StartTag {
-    my ($expat, $element) = @_;
-    if ($element eq "dublin_core") {
-	$self->{'saved_metadata'} = {};
-    } elsif ($element eq "dcvalue") {
-	my $metaname = $_{'element'};
-	my $qualifier = $_{'qualifier'}||"";
-	if ($metaname ne "description" || $qualifier ne "provenance") {
-	    $metaname .= "^$qualifier" if ($qualifier ne "none" && $qualifier ne "");
-	    $self->{'metaname'} = "ex.dc.\u$metaname";
-	}
-    }
-}
-
-sub EndTag {
-    my ($expat, $element) = @_;
-    
-    if ($element eq "dcvalue") {
-	$self->{'metaname'} = "";
-    }
-}
-
-sub Text {
-    if (defined ($self->{'metaname'}) && $self->{'metaname'} ne "") {
-	# $_ == Metadata content
-	my $mname = $self->{'metaname'};
-	my $mvalue = prepareMetadataValue($_);
-	if (defined $self->{'saved_metadata'}->{$mname}) {
-	    # accumulate - add value to existing value(s)
-	    if (ref ($self->{'saved_metadata'}->{$mname}) eq "ARRAY") {
-		push (@{$self->{'saved_metadata'}->{$mname}}, $mvalue);
-	    } else {
-		$self->{'saved_metadata'}->{$mname} = 
-		    [$self->{'saved_metadata'}->{$mname}, $mvalue];
-	    }
-	} else {
-	    # accumulate - add value into (currently empty) array
-	    $self->{'saved_metadata'}->{$mname} = [$mvalue];
-	}
-
-    }
-}
-
-# Prepare DSpace metadata for using with Greenstone.
-# Some value must be escaped.
-sub prepareMetadataValue {
-	my ($value) = @_;
-	
-	$value =~ s/\[/&#091;/g;
-	$value =~ s/\]/&#093;/g;
-	
-	return $value;
- }
-# This Char function overrides the one in XML::Parser::Stream to overcome a
-# problem where $expat->{Text} is treated as the return value, slowing
-# things down significantly in some cases.
-sub Char {
-    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
-    $_[0]->{'Text'} .= $_[1];
-    return undef;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/DatabasePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DatabasePlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,379 +1,0 @@
-###########################################################################
-#
-# DatabasePlugin.pm -- plugin to import records from a database
-# 
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2003 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# A plugin that imports records from a database. This uses perl's DBI module, 
-# which includes back-ends for mysql, postgresql, comma separated values (CSV),
-# MS Excel, ODBC, sybase, etc... Extra modules may need to be installed to 
-# use this. See <GSDLHOME>/etc/packages/example.dbi for an example config file.
-#
-
-# Written by John McPherson for the NZDL project
-# Mar, Apr 2003
-
-package DatabasePlugin;
-
-use strict;
-no strict 'refs'; # allow variable as a filehandle
-
-use AutoExtractMetadata;
-use MetadataRead;
-use unicode;
-
-sub BEGIN {
-    @DatabasePlugin::ISA = ('MetadataRead', 'AutoExtractMetadata');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" }];
-
-my $options = { 'name'     => "DatabasePlugin",
-		'desc'     => "{DatabasePlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new AutoExtractMetadata($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.dbi$^;
-}
-
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs,$total_count,$gli) = @_;
-        
-     #see if we can handle the passed file...
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-    
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-
-    print STDERR "<Processing n='$file' p='DatabasePlugin'>\n" if ($gli);
-    print $outhandle "DatabasePlugin: processing $file\n" 
-	if $self->{'verbosity'} > 1;
-   
-    require DBI; # database independent stuff
-
-    # calculate the document hash, for document ids
-    my $hash="0";
-
-   
-    # default options - may be overridden by config file
-    my $language=undef;
-    my $encoding=undef;
-    my $dbplug_debug=0;
-    my $username='';
-    my $password='';
-
-    # these settings must be set by the config file:
-    my $db=undef;
-
-    # get id of pages from "nonempty", get latest version number from 
-    # "recent", and then get pagename from "page" and content from "version" !
-
-    my $sql_query_prime = undef ;
-    my $sql_query = undef ;
-
-    my %db_to_greenstone_fields=();
-    my %callbacks=();
-
-
-    # read in config file.
-    if (!open (CONF, $filename_full_path)) {
-	    print $outhandle "DatabasePlugin: can't read $filename_full_path: $!\n";
-	    return 0;
-    } 
-   
-    my $line;
-    my $statement="";
-    my $callback="";
-    while (defined($line=<CONF>)) {
-	chomp $line;
-	$line .= " "; # for multi-line statements - don't conjoin!
-	$line =~ s/\s*\#.*$//mg; # remove comments
-	$statement .= $line;
-
-	if ($line =~ /^\}\s*$/ && $callback) { # ends the callback
-	    $callback .= $statement ; $statement = "";
-	    # try to check that the function is "safe"
-	    if ($callback  =~ /\b(?:system|open|pipe|readpipe|qx|kill|eval|do|use|require|exec|fork)\b/ ||
-		$callback =~ /[\`]|\|\-/) {
-		# no backticks or functions that start new processes allowed
-		print $outhandle "DatabasePlugin: bad function in callback\n";
-		return 0;
-	    }
-	    $callback =~ s/sub (\w+?)_callback/sub/;
-	    my $fieldname = $1;
-	    my $ret = eval "\$callbacks{'$fieldname'} = $callback ; 1";
-	    if (!defined($ret)) {
-		print $outhandle "DatabasePlugin: error eval'ing callback: $@\n";
-		exit(1);
-	    }
-	    $callback="";
-	    print $outhandle "DatabasePlugin: callback registered for '$fieldname'\n"
-	        if $dbplug_debug;
-	} elsif ($callback) {
-	    # add this line to the callback function
-	    $callback .= $statement;
-	    $statement = "";
-	} elsif ($statement =~ m/;\s*$/) { # ends with ";"
-	    # check that it is safe
-	    # assignment
-	    if ($statement =~ m~(\$\w+)\s* = \s*
-		(\d		# digits
-		 | ".*?(?<!\\)" # " up to the next " not preceded by a \
-		 | '.*?(?<!\\)' # ' up to the next ' not preceded by a \
-		)\s*;~x ||      # /x means ignore comments and whitespace in rx
-		$statement =~ m~(\%\w+)\s*=\s*(\([\w\s\"\',:=>]+\))\s*;~ ) {   
-		# evaluate the assignment, return 1 on success "
-		if (!eval "$1=$2; 1") {
-		    my $err=$@;
-		    chomp $err;
-		    $err =~ s/\.$//; # remove a trailing .
-		    print $outhandle "DatabasePlugin: error evaluating `$statement'\n";
-		    print $outhandle " $err (in $filename_full_path)\n";
-		    return 0; # there was an error reading the config file
-		}
-	    } elsif ($statement =~ /sub \w+_callback/) {
-		# this is the start of a callback function definition
-		$callback = $statement;
-		$statement = "";
-	    } else {
-		print $outhandle "DatabasePlugin: skipping statement `$statement'\n";
-	    }
-	    $statement = "";
-	}
-    }
-    close CONF;
-
-    
-    if (!defined($db)) {
-	print $outhandle "DatabasePlugin: error: $filename_full_path does not specify a db!\n";
-	return 0;
-    }
-    if (!defined($sql_query)) {
-    	print $outhandle "DatabasePlugin: error: no SQL query specified!\n";
-	return 0;
-    }
-    # connect to database
-    my $dbhandle=DBI->connect($db, $username, $password);
-
-    if (!defined($dbhandle)) {
-	die "DatabasePlugin: could not connect to database, exiting.\n";
-    }
-    if (defined($dbplug_debug) && $dbplug_debug==1) {
-	print $outhandle "DatabasePlugin (debug): connected ok\n";
-    }
-
-    my $statement_hand;
-
-    # The user gave 2 sql statements to execute?
-    if ($sql_query_prime) {
-	    $statement_hand=$dbhandle->prepare($sql_query_prime);
-	    $statement_hand->execute;
-	    if ($statement_hand->err) {
-		    print $outhandle "Error: " . $statement_hand->errstr . "\n";
-		    return undef;
-	    }
-    }
-
-  
-    $statement_hand=$dbhandle->prepare($sql_query);
-    $statement_hand->execute;
-    if ($statement_hand->err) {
-    	print $outhandle "Error:" . $statement_hand->errstr . "\n";
-	return undef;
-    }
-
-    # get the array-ref for the field names and cast it to array
-    my @field_names;
-    @field_names=@{ $statement_hand->{NAME} };
-
-    foreach my $fieldname (@field_names) {
-	if (defined($db_to_greenstone_fields{$fieldname})) {
-	    if (defined($dbplug_debug) && $dbplug_debug==1) {
-		print $outhandle "DatabasePlugin (debug): mapping db field "
-		    . "'$fieldname' to "
-			. $db_to_greenstone_fields{$fieldname} . "\n";
-	    }
-	    $fieldname=$db_to_greenstone_fields{$fieldname};
-	}
-    }
-
-    # get rows
-
-    my $count = 0;
-    my @row_array;
-
-    @row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
-
-    my $base_oid = undef;
-    while (scalar(@row_array)) {
-	if (defined($dbplug_debug) && $dbplug_debug==1) {
-	    print $outhandle "DatabasePlugin (debug): retrieved a row from query\n";
-	}
-
-	# create a new document
-	my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-
-	my $cursection = $doc_obj->get_top_section();
-
-	# if $language not set in config file, will use BasePlugin's default
-	if (defined($language)) {
-	    $doc_obj->add_utf8_metadata($cursection, "Language", $language);
-	}
-	# if $encoding not set in config file, will use BasePlugin's default
-	if (defined($encoding)) {
-	    # allow some common aliases
-	    if ($encoding =~ m/^utf[-_]8$/i) {$encoding="utf8"}
-	    $encoding =~ s/-/_/g; # greenstone uses eg iso_8859_1
-	    $doc_obj->add_utf8_metadata($cursection, "Encoding", $encoding);
-	}
-
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-	my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-	$self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-
-	if ($self->{'cover_image'}) {
-	    $self->associate_cover_image($doc_obj, $filename_full_path);
-	}
-	$doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-
-	$doc_obj->add_metadata($doc_obj->get_top_section(), "FileFormat", "DB");
-
-	# include any metadata passed in from previous plugins 
-	# note that this metadata is associated with the top level section
-	$self->extra_metadata ($doc_obj, $cursection,
-			       $metadata);
-
-	# do any automatic metadata extraction
-	$self->auto_extract_metadata ($doc_obj);
-
-	my $unique_id=undef;
-
-	foreach my $fieldname (@field_names) {
-	    my $fielddata=shift @row_array;
-
-	    if (! defined($fielddata) ) {
-	    	next; # this field was "" or NULL
-	    }
-	    # use the specified encoding, defaulting to utf-8
-	    if (defined($encoding) && $encoding ne "ascii"
-	    	&& $encoding ne "utf8") {
-	      $fielddata=&unicode::unicode2utf8(
-	      	&unicode::convert2unicode($encoding, \$fielddata)
-					       );
-	    }
-	    # see if we have a ****_callback() function defined
-	    if (exists $callbacks{$fieldname}) {
-		my $funcptr = $callbacks{$fieldname};
-		$fielddata = &$funcptr($fielddata);
-	    }
-
-	    if ($fieldname eq "text") {
-		# add as document text
-		$fielddata=~s@<@&lt;@g;
-		$fielddata=~s@>@&gt;@g; # for xml protection...
-		$fielddata=~s@_@\\_@g; # for macro language protection...
-		$doc_obj->add_utf8_text($cursection, $fielddata);
-	    } elsif ($fieldname eq "Identifier") {
-		# use as greenstone's unique record id
-		if ($fielddata =~ /^\d+$/) {
-		    # don't allow IDs that are completely numeric
-		    $unique_id="id" . $fielddata;
-		} else {
-		    $unique_id=$fielddata;
-		}
-	    } else {
-		# add as document metadata
-		$fielddata=~s/\[/&#91;/g;
-		$fielddata=~s/\]/&#93;/g;
-		$doc_obj->add_utf8_metadata($cursection,
-					    $fieldname, $fielddata);
-
-	    }
-	}
-
-
-	if (!defined $unique_id) {
-	    if (!defined $base_oid) {
-		$self->add_OID($doc_obj);
-		$base_oid = $doc_obj->get_OID(); 
-	    }
-	    $doc_obj->set_OID($base_oid."s$count");
-	} else {
-	    # use our id from the database...
-	    $doc_obj->set_OID($unique_id);
-	}
-
-
-        # process the document
-	$processor->process($doc_obj);
-
-	$count++;
-
-	# get next row
-	@row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
-    } # end of row_array is not empty
-
-    # check "$sth->err" if empty array for error
-    if ($statement_hand->err) {
-	print $outhandle "DatabasePlugin: received error: \"" .
-	    $statement_hand->errstr . "\"\n";
-    }
-
-    # clean up connection to database
-    $statement_hand->finish();
-    $dbhandle->disconnect();
-
-    # num of input files, rather than documents created?
-    $self->{'num_processed'}++;
-
-    if (defined($dbplug_debug) && $dbplug_debug==1) {
-        print $outhandle "DatabasePlugin: imported $count DB records as documents.\n";
-    }
-    $count;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/GreenstoneXMLPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/GreenstoneXMLPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,351 +1,0 @@
-
-###########################################################################
-#
-# GreenstoneXMLPlugin.pm
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Processes GreenstoneArchive XML documents. Note that this plugin does no
-# syntax checking (though the XML::Parser module tests for
-# well-formedness). It's assumed that the GreenstoneArchive files conform
-# to their DTD.
-
-package GreenstoneXMLPlugin;
-
-use Encode;
-use File::Basename;
-
-use ReadXMLFile;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @GreenstoneXMLPlugin::ISA = ('ReadXMLFile');
-}
-
-
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)doc\.xml$^;
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "GreenstoneXMLPlugin",
-		'desc'     => "{GreenstoneXMLPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-	        'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    $self->{'section'} = "";
-    $self->{'section_level'} = 0;
-    $self->{'metadata_name'} = "";
-    $self->{'metadata_value'} = "";
-    $self->{'content'} = "";
-    $self->{'metadata_read_store'} = {};
-
-#    # Currently used to store information for previous values controls. In
-#    # the next contract I'll move to using information directly from Lucene.
-#    $self->{'sqlfh'} = 0;
-   
-    return bless $self, $class;
-}
-
-
-
-
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-    
-    print $outhandle "GreenstoneXMLlugin: setting up block list for $file\n"
-	if $self->{'verbosity'} > 1;
-
-    my $line;
-    if (open(GIN,"<:utf8",$filename_full_path)) {
-
-	while (defined($line=<GIN>)) {
-	    if ($line =~ m@<Metadata\s+name="gsdlassocfile">([^:]*):(?:[^:]*):(?:[^:]*)</Metadata>@) {
-		my $gsdl_assoc_file = $1;
-
-		my $dirname = dirname($filename_full_path);
-		my $full_gsdl_assoc_filename = &util::filename_cat($dirname,$gsdl_assoc_file);
-		if ($self->{'verbosity'}>2) {
-		    print $outhandle "  Storing block list item: $full_gsdl_assoc_filename\n";
-		}
-
-		&util::block_filename($block_hash,$full_gsdl_assoc_filename);		
-	    }
-	}
-
-	close(GIN);
-    }
-    else {
-	
-	print $outhandle "Error: Failed to open $file in GreenstoneXMLPlugin::metadata_read()\n";
-	print $outhandle "       $!\n";
-    }
-
-
-    $self->{'metadata_read_store'}->{$filename_full_path} = 1;
-
-    return 1;
-}
-
-
-sub xml_start_document {
-}
-
-sub xml_end_document {
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "(Greenstone)?Archive";
-}
-
-
-sub xml_doctype {
-    my $self = shift(@_);
-
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    # allow the short-lived and badly named "GreenstoneArchive" files to be processed
-    # as well as the "Archive" files which should now be created by import.pl
-    die "" if ($name !~ /^(Greenstone)?Archive$/);
-
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "GreenstoneXMLPlugin: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
-    print STDERR "<Processing n='$self->{'file'}' p='GreenstoneXMLPlugin'>\n" if $self->{'gli'};
-
-}
-
-
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    $self->{'element'} = $element;
-    if ($element eq "Section") {
-	if ($self->{'section_level'} == 0) {
-	    $self->open_document();
-       	} else {
-	    my $doc_obj = $self->{'doc_obj'};
-	    $self->{'section'} = 
-		$doc_obj->insert_section($doc_obj->get_end_child($self->{'section'}));
-	}
-	
-	$self->{'section_level'} ++;
-    }
-    elsif ($element eq "Metadata") {
-	$self->{'metadata_name'} = $_{'name'};
-    }
-}
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    if ($element eq "Section") {
-	$self->{'section_level'} --;
-	$self->{'section'} = $self->{'doc_obj'}->get_parent_section ($self->{'section'});
-	$self->close_document() if $self->{'section_level'} == 0;
-    }
-    elsif ($element eq "Metadata") {
-	# text read in by XML::Parser is in Perl's binary byte value
-	# form ... need to explicitly make it UTF-8
-
-
-	my $metadata_name = decode("utf-8",$self->{'metadata_name'});
-	my $metadata_value = decode("utf-8",$self->{'metadata_value'});
-
-	$self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, 
-					      $metadata_name,$metadata_value);
-
-        # Ensure this value is added to the allvalues database in gseditor.
-        # Note that the database constraints prevent multiple occurances of the
-        # same key-value pair.
-        # We write these out to a file, so they can all be commited in one
-        # transaction
-        #if (!$self->{'sqlfh'})
-        #  {
-        #    my $sql_file = $ENV{'GSDLHOME'} . "/collect/lld/tmp/gseditor.sql";
-        #    # If the file doesn't already exist, open it and begin a transaction
-        #    my $sql_fh;
-        #    if (!-e $sql_file)
-        #      {
-        #        open($sql_fh, ">" . $sql_file);
-        #        print $sql_fh "BEGIN TRANSACTION;\n";
-        #      }
-        #    else
-        #      {
-        #        open($sql_fh, ">>" . $sql_file);
-        #      }
-        #    print STDERR "Opened SQL log\n";
-        #    $self->{'sqlfh'} = $sql_fh;
-        #  }
-
-        #my $mvalue = $self->{'metadata_value'};
-        #$mvalue =~ s/\'/\'\'/g;
-        #$mvalue =~ s/_claimantsep_/ \& /g;
-
-        #my $fh = $self->{'sqlfh'};
-        #if ($fh)
-        #  {
-        #    print $fh "INSERT INTO allvalues (mkey, mvalue) VALUES ('" . $self->{'metadata_name'} . "', '" . $mvalue . "');\n";
-        #  }
-
-        # Clean Up
-	$self->{'metadata_name'} = "";
-	$self->{'metadata_value'} = "";
-    }
-    elsif ($element eq "Content" && $self->{'content'} ne "") {
-
-	# text read in by XML::Parser is in Perl's binary byte value
-	# form ... need to explicitly make it UTF-8
-	my $content = decode("utf-8",$self->{'content'});
-
-	$self->{'doc_obj'}->add_utf8_text($self->{'section'}, $content);
-	$self->{'content'} = "";
-    }
-    $self->{'element'} = "";
-}
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-
-    if ($self->{'element'} eq "Metadata") {
-	$self->{'metadata_value'} .= $_;
-    }
-    elsif ($self->{'element'} eq "Content") {
-	$self->{'content'} .= $_;
-    }
-}
-
-sub open_document {
-    my $self = shift(@_);
-
-    my $filename = $self->{'filename'};
-
-    # create a new document
-    if (defined $self->{'metadata_read_store'}->{$filename}) {
-	# Being processed as part of *import* phase 
-	# (i.e. was in import directory)
-	$self->SUPER::open_document(@_);
-	delete $self->{'metadata_read_store'}->{$filename};
-    }
-    else {
-	# Otherwise being processed as part of the *buildcol* phase 
-	# (i.e. named directly by ArchiveInf plugin)
-	$self->{'doc_obj'} = new doc();
-    }
-
-    $self->{'section'} = "";
-}
-
-sub close_document {
-    my $self = shift(@_);
-
-    # add the associated files
-    my $assoc_files = 
-	$self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
-
-    # for when "assocfilepath" isn't the same directory that doc.xml is in...
-    my $assoc_filepath_list= $self->{'doc_obj'}->get_metadata($self->{'doc_obj'}->get_top_section(), "assocfilepath");
-
-    my $assoc_filepath=shift (@$assoc_filepath_list);
-
-    #rint STDERR "Filename is: " . $self->{'filename'} . "\n";
-    #rint STDERR "Initially my assoc_filepath is: $assoc_filepath\n";
-    #rint STDERR "Custom archive dir is: " . $self->{'base_dir'} . "\n";
-    # Correct the assoc filepath if one is defined
-    if (defined ($assoc_filepath))
-      {
-        # Check whether the assoc_filepath already includes the base dir
-        if (index($assoc_filepath, $self->{'base_dir'}) == -1)
-          {
-            # And if not, append it so as to make this absolute
-            $assoc_filepath = &util::filename_cat($self->{'base_dir'}, $assoc_filepath);
-          }
-      }
-    else
-      {
-	$assoc_filepath = $self->{'filename'};
-	$assoc_filepath =~ s/[^\\\/]*$//;
-      }
-    #rint STDERR "Goned and made it absolute: $assoc_filepath\n";
-
-    foreach my $assoc_file_info (@$assoc_files) {
-	my ($assoc_file, $mime_type, $dir) = split (":", $assoc_file_info);
-        #rint STDERR "assoc_file: $assoc_file\n";
-        #rint STDERR "mime_type: $mime_type\n";
-        #rint STDERR "dir: $dir\n";
-	my $real_dir = &util::filename_cat($assoc_filepath, $assoc_file),
-	my $assoc_dir = (defined $dir && $dir ne "") 
-	    ? &util::filename_cat($dir, $assoc_file) : $assoc_file;
-	$self->{'doc_obj'}->associate_file($real_dir, $assoc_dir, $mime_type);
-        #rint STDERR "According to me the real assoc_filepath is: $real_dir\n";
-    }
-    $self->{'doc_obj'}->delete_metadata($self->{'doc_obj'}->get_top_section(), "gsdlassocfile");
-
-    # process the document
-    $self->{'processor'}->process($self->{'doc_obj'}, $self->{'file'});
-
-    $self->{'num_processed'} ++;
-    undef $self->{'doc_obj'};
-}
-
-
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/HBPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/HBPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,332 +1,0 @@
-###########################################################################
-#
-# HBPlugin.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# plugin which processes an HTML book directory
-
-# This plugin is used by the Humanity Library collections and does not handle
-# input encodings other than ascii or extended ascii
-
-# this code is kind of ugly and could no doubt be made to run faster, by leaving
-# it in this state I hope to encourage people to make their collections use
-# HBSPlug instead ;-)
-
-# Use HBSPlug if creating a new collection and marking up files like the
-# Humanity Library collections. HBSPlug accepts all input encodings but
-# expects the marked up files to be cleaner than those used by the
-# Humanity Library collections
-
-package HBPlugin;
-
-use ghtml;
-use BasePlugin;
-use unicode;
-use util;
-use doc;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @HBPlugin::ISA = ('BasePlugin');
-}
-my $encoding_list =     
-    [ { 'name' => "ascii",
-	'desc' => "{BasePlugin.encoding.ascii}" },
-      { 'name' => "iso_8859_1",
-	'desc' => "{HBPlugin.encoding.iso_8859_1}" } ];
- 
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "input_encoding",
-	'desc' => "{ReadTextFile.input_encoding}",
-	'type' => "enum",
-	'deft' => "iso_8859_1",
-	'list' => $encoding_list,
-	'reqd' => "no" }
-      ];
-
-my $options = { 'name'     => "HBPlugin",
-		'desc'     => "{HBPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-# this is included only to prevent warnings being printed out 
-# from BasePlugin::init. The process_exp is not used by this plugin
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return "This plugin does not use a process_exp\n";
-}
-
-
-sub HB_read_html_file {
-    my $self = shift (@_);
-    my ($htmlfile, $text) = @_;
-
-    # load in the file
-    if (!open (FILE, $htmlfile)) {
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "ERROR - could not open $htmlfile\n";
-	return;
-    }
-
-    my $foundbody = 0;
-    $self->HB_gettext (\$foundbody, $text, "FILE");
-    close FILE;
-
-    # just in case there was no <body> tag
-    if (!$foundbody) {
-	$foundbody = 1;
-	open (FILE, $htmlfile) || return;
-	$self->HB_gettext (\$foundbody, $text, "FILE");	
-	close FILE;
-    }
-    # text is in utf8
-}
-
-# converts the text to utf8, as ghtml does that for &eacute; etc.
-sub HB_gettext {
-    my $self = shift (@_);
-    my ($foundbody, $text, $handle) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    my $line = "";
-    while (defined ($line = <$handle>)) {
-	# look for body tag
-	if (!$$foundbody) {
-	    if ($line =~ s/^.*<body[^>]*>//i) {
-		$$foundbody = 1;
-	    } else {
-		next;
-	    }
-	}
-	
-	# check for symbol fonts
-	if ($line =~ /<font [^>]*?face\s*=\s*\"?(\w+)\"?/i) {
-	    my $font = $1;
-	    print $outhandle "HBPlugin::HB_gettext - warning removed font $font\n" 
-		if ($font !~ /^arial$/i);
-	}
-
-	$line =~ s/<\/p>//ig;   # remove </p> tags
-	$line =~ s/<\/?(body|html|font)\b[^>]*>//ig; # remove any unwanted tags
-
-	$$text .= $line;
-    }
-    #
-    if ($self->{'input_encoding'} eq "iso_8859_1") {
-	# convert to utf-8
-	$$text=&unicode::unicode2utf8(&unicode::convert2unicode("iso_8859_1", $text));
-    }
-    # convert any alphanumeric character entities to their utf-8
-    # equivalent for indexing purposes
-    &ghtml::convertcharentities ($$text);
-
-    $$text =~ s/\s+/ /g; # remove \n's
-}
-
-sub HB_clean_section {
-    my $self = shift (@_);
-    my ($section) = @_;
-
-    # remove tags without a starting tag from the section
-    my ($tag, $tagstart);
-    while ($section =~ /<\/([^>]{1,10})>/) {
-	$tag = $1;
-	$tagstart = index($section, "<$tag");
-	last if (($tagstart >= 0) && ($tagstart < index($section, "<\/$tag")));
-	$section =~ s/<\/$tag>//;
-    }
-    
-    # remove extra paragraph tags
-    while ($section =~ s/<p\b[^>]*>\s*<p\b/<p/ig) {}
-    
-    # remove extra stuff at the end of the section
-    while ($section =~ s/(<u>|<i>|<b>|<p\b[^>]*>|&nbsp;|\s)$//i) {}
-    
-    # add a newline at the beginning of each paragraph
-    $section =~ s/(.)\s*<p\b/$1\n\n<p/gi;
-    
-    # add a newline every 80 characters at a word boundary
-    # Note: this regular expression puts a line feed before
-    # the last word in each section, even when it is not
-    # needed.
-    $section =~ s/(.{1,80})\s/$1\n/g;
-    
-    # fix up the image links
-    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/
-	<center><img src=\"_httpdocimg_\/$1\"><\/center><br>/ig;
-    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/
-	<center><img src=\"_httpdocimg_\/$1\"><\/center><br>/ig;
-
-    return $section;
-}
-
-
-sub shorten {
-    my $self = shift (@_);
-    my ($text) = @_;
-
-    return "\"$text\"" if (length($text) < 100);
-
-    return "\"" . substr ($text, 0, 50) . "\" ... \"" . 
-	substr ($text, length($text)-50) . "\"";
-}
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # get the html filename and see if this is an HTML Book...
-    my $jobnumber = $file;
-    if ($file =~ /[\\\/]/) {
-	($jobnumber) = $file =~ /[\\\/]([^\\\/]+)$/;
-    }
-    return undef unless defined $jobnumber;
-    my $htmlfile = &util::filename_cat($base_dir, $file, "$jobnumber.htm");
-    return undef unless -e $htmlfile;
-
-    print STDERR "<Processing n='$file' p='HBPlugin'>\n" if ($gli);
-    print $outhandle "HBPlugin: processing $file\n";
-
-    # read in the file and do basic html cleaning (removing header etc)
-    my $html = "";
-    $self->HB_read_html_file ($htmlfile, \$html);
-    # html is in utf8
-
-    # create a new document
-    my $doc_obj = new doc ($file, "indexed_doc", $self->{'file_rename_method'});
-
-    # copy the book cover if it exists
-    my $bookcover = &util::filename_cat($base_dir, $file, "$jobnumber.jpg");
-    $doc_obj->associate_file($bookcover, "cover.jpg", "image/jpeg");
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileFormat", "HB");
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "FileSize", (-s $htmlfile));
-
-    my $cursection = $doc_obj->get_top_section();
-    
-    # add metadata for top level of document
-    foreach my $field (keys(%$metadata)) {
-	# $metadata->{$field} may be an array reference
-	if (ref ($metadata->{$field}) eq "ARRAY") {
-	    map {
-		$doc_obj->add_utf8_metadata($cursection, $field, $_);
-	    } @{$metadata->{$field}};
-	} else {
-	    $doc_obj->add_utf8_metadata($cursection, $field, $metadata->{$field}); 
-	}
-    }
-
-    # process the file one section at a time
-    my $curtoclevel = 1;
-    my $firstsection = 1;
-    while (length ($html) > 0) {
-	if ($html =~ s/^.*?(?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC(\d+)&gt;&gt;\s*(.*?)<p\b/<p/i) {
-	    my $toclevel = $3;
-	    my $title = $4;
-	    my $sectiontext = "";
-	    if ($html =~ s/^(.*?)((?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC\d+&gt;&gt;)/$2/i) {
-		$sectiontext = $1;
-	    } else {
-		$sectiontext = $html;
-		$html = "";
-	    }
-
-	    # remove tags and extra spaces from the title
-	    $title =~ s/<\/?[^>]+>//g;
-	    $title =~ s/^\s+|\s+$//g;
-
-	    # close any sections below the current level and
-	    # create a new section (special case for the firstsection)
-	    while (($curtoclevel > $toclevel) ||
-		   (!$firstsection && $curtoclevel == $toclevel)) {
-		$cursection = $doc_obj->get_parent_section ($cursection);
-		$curtoclevel--;
-	    }
-	    if ($curtoclevel+1 < $toclevel) {
-		print $outhandle "WARNING - jump in toc levels in $htmlfile " . 
-		    "from $curtoclevel to $toclevel\n";
-	    }
-	    while ($curtoclevel < $toclevel) {
-		$curtoclevel++;
-		$cursection = 
-		    $doc_obj->insert_section($doc_obj->get_end_child($cursection));
-	    }
-
-	    # add the metadata to this section
-	    $doc_obj->add_utf8_metadata($cursection, "Title", $title);
-
-	    # clean up the section html
-	    $sectiontext = $self->HB_clean_section($sectiontext);
-
-	    # associate any files
-	    map { $doc_obj->associate_file(&util::filename_cat ($base_dir, $file, $1), $1)
-		      if /_httpdocimg_\/([^\"]+)\"/; 0; }
-	         split (/(_httpdocimg_\/[^\"]+\")/, $sectiontext);
-
-	    # add the text for this section
-	    $doc_obj->add_utf8_text ($cursection, $sectiontext);
-	} else {
-	    print $outhandle "WARNING - leftover text\n" , $self->shorten($html), 
-	    "\nin $htmlfile\n";
-	    last;
-	}
-	$firstsection = 0;
-    }
-
-    # add a OID
-    $self->add_OID($doc_obj);
-
-    # process the document
-    $processor->process($doc_obj, &util::filename_cat($file, "$jobnumber.htm"));
-        
-    return 1; # processed the file
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/KeyphraseExtractor.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/KeyphraseExtractor.pm	(revision 26943)
+++ 	(revision )
@@ -1,145 +1,0 @@
-###########################################################################
-#
-# KeyphraseExtractor - helper plugin to extract key phrases
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package KeyphraseExtractor;
-
-use Kea;
-use PrintInfo;
-use gsprintf 'gsprintf';
-
-use strict;
-no strict 'subs';
-
-BEGIN {
-    @KeyphraseExtractor::ISA = ('PrintInfo');
-}
-
-my $arguments = [
-      { 'name' => "extract_keyphrases",
-	'desc' => "{KeyphraseExtractor.extract_keyphrases}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "extract_keyphrases_kea4",
-	'desc' => "{KeyphraseExtractor.extract_keyphrases_kea4}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "extract_keyphrase_options",
-	'desc' => "{KeyphraseExtractor.extract_keyphrase_options}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no" }
-		 ];
-
-my $options = { 'name'     => "KeyphraseExtractor",
-		'desc'     => "{KeyphraseExtractor.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists, 1);
-
-    return bless $self, $class;
-
-}
-# extract metadata
-sub extract_keyphrase_metadata {
-
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-
-    if ($self->{'extract_keyphrases'} || $self->{'extract_keyphrases_kea4'}) {
-	$self->extract_keyphrases($doc_obj);
-    }
-
-}
-
-
-#adding kea keyphrases
-sub extract_keyphrases
-{
-    my $self = shift(@_);
-    my $doc_obj = shift(@_);
-
-    # Use Kea 3.0 unless 4.0 has been specified
-    my $kea_version = "3.0";
-    if ($self->{'extract_keyphrases_kea4'}) {
-	$kea_version = "4.0";
-    }
-
-    # Check that Kea exists, and tell the user where to get it if not
-    my $keahome = &Kea::get_Kea_directory($kea_version);
-    if (!-e $keahome) {
-	gsprintf(STDERR, "{KeyphraseExtractor.missing_kea}\n", $keahome, $kea_version);
-	return;
-    }
-
-    my $thissection = $doc_obj->get_top_section();
-    my $text = "";
-    my $list;
-
-    #loop through sections to gather whole doc
-    while (defined $thissection) { 
-	my $sectiontext = $doc_obj->get_text($thissection);   
-	$text = $text.$sectiontext;
-	$thissection = $doc_obj->get_next_section ($thissection);
-    } 
-   
-    if($self->{'extract_keyphrase_options'}) { #if kea options flag is set, call Kea with specified options 
-	$list = &Kea::extract_KeyPhrases ($kea_version, $text, $self->{'extract_keyphrase_options'});
-    } else { #otherwise call Kea with no options
-	$list = &Kea::extract_KeyPhrases ($kea_version, $text);
-    }
- 
-    if ($list){
-	# if a list of kea keyphrases was returned (ie not empty)
-	if ($self->{'verbosity'}) {
-	    gsprintf(STDERR, "{KeyphraseExtractor.keyphrases}: $list\n");
-	}
-
-	#add metadata to top section
-	$thissection = $doc_obj->get_top_section(); 
-
-	# add all key phrases as one metadata
-	$doc_obj->add_metadata($thissection, "Keyphrases", $list);
-
-	# add individual key phrases as multiple metadata
-	foreach my $keyphrase (split(',', $list)) {
-	    $keyphrase =~ s/^\s+|\s+$//g;
-	    $doc_obj->add_metadata($thissection, "Keyphrase", $keyphrase);
-	}
-    }
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MP3Plugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MP3Plugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,369 +1,0 @@
-###########################################################################
-#
-# MP3Plugin.pm -- Plugin for MP3 files (MPEG audio layer 3).
-#
-# A component of the Greenstone digital library software from the New
-# Zealand Digital Library Project at the University of Waikato, New
-# Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package MP3Plugin;
-
-use BasePlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-use MP3::Info;
-
-require giget;
-
-sub BEGIN {
-    @MP3Plugin::ISA = ('BasePlugin');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "assoc_images",
-        'desc' => "{MP3Plugin.assoc_images}",
-        'type' => "flag",
-        'deft' => "",
-        'reqd' => "no" },
-      { 'name' => "applet_metadata",
-	'desc' => "{MP3Plugin.applet_metadata}",
-	'type' => "flag",
-	'deft' => "" },
-      { 'name' => "metadata_fields",
-	'desc' => "{MP3Plugin.metadata_fields}",
-	'type' => "string",
-	'deft' => "Title,Artist,Genre" },
-      { 'name' => "file_rename_method",
-	'desc' => "{BasePlugin.file_rename_method}",
-	'type' => "enum",
-	'deft' => &get_default_file_rename_method(), # by default rename imported files and assoc files using this encoding
-	'list' => $BasePlugin::file_rename_method_list,
-	'reqd' => "no"
-      } ];
-
-my $options = { 'name'     => "MP3Plugin",
-		'desc'     => "{MP3Plugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    return q^(?i)\.mp3$^;
-}
-
-# rename imported files and assoc files using base64 encoding by default
-# so that the urls generated will always work with media files even when
-# opened in external applications (when wmv file names contain spaces in
-# them and they get url-encoded, wmv player fails to open the doubly url-
-# encoded url reference to the url-encoded filename). 
-sub get_default_file_rename_method() {
-    return "base64";
-}
-
-# we don't want to hash on the file
-sub get_oid_hash_type {
-    my $self = shift (@_);
-    return "hash_on_ga_xml";
-}
-
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    # associate the file with the document
-    if ($self->associate_mp3_file($filename_full_path, $filename_no_path, $doc_obj) != 1)
-    {
-	print "MP3Plugin: couldn't process \"$filename_full_path\"\n";
-	return 0;
-    }
-   
-    my $text = &gsprintf::lookup_string("{BasePlugin.dummy_text}",1);
-    if ($self->{'assoc_images'}) {
-	$text .= "[img1]<br>";
-	$text .= "[img2]<br>";
-    }
-    $doc_obj->add_utf8_text($doc_obj->get_top_section(), $text);
-    $doc_obj->add_metadata ($doc_obj->get_top_section(), "NoText",    "1");
-
-}
-
-sub gen_mp3applet {
-
-    my ($mp3_filename) = @_;
-
-    my $applet_html = '
-<OBJECT classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
-        WIDTH = "59"
-        HEIGHT = "32"
-        codebase="http://java.sun.com/products/plugin/1.3/jinstall-13-win32.cab#Version=1,3,0,0">
-<PARAM NAME = CODE VALUE = "javazoom.jlGui.TinyPlayer" >
-<PARAM NAME = ARCHIVE VALUE = "_httpcollection_/tinyplayer.jar,_httpcollection_/jl10.jar" >
-<PARAM NAME="type" VALUE="application/x-java-applet;version=1.3">
-<PARAM NAME="scriptable" VALUE="false">
-<PARAM NAME = "skin" VALUE ="_httpcollection_/skins/Digitalized">
-<PARAM NAME = "autoplay" VALUE ="yes">
-<PARAM NAME = "bgcolor" VALUE ="638182">
-<PARAM NAME = "audioURL" VALUE ="MP3FILENAME">
-<COMMENT>
-<EMBED type="application/x-java-applet;version=1.3"
-       CODE = "javazoom.jlGui.TinyPlayer"
-       ARCHIVE = "_httpcollection_/tinyplayer.jar,_httpcollection_/jl10.jar"
-       WIDTH = "59"
-       HEIGHT = "32"
-       skin = "_httpcollection_/skins/Digitalized"
-       autoplay = "yes"
-       bgcolor = "638182"
-       audioURL = "MP3FILENAME"
-       scriptable=false
-       pluginspage="http://java.sun.com/products/plugin/1.3/plugin-install.html">
-<NOEMBED>
-</COMMENT>
-</NOEMBED></EMBED>
-</OBJECT>
-';
-
-    $applet_html =~ s/MP3FILENAME/$mp3_filename/g;
-
-    return $applet_html;
-}
-
-
-
-# Associate the mp3 file with the new document
-
-sub associate_mp3_file {
-    my $self = shift (@_);
-    my $filename = shift (@_);   # filename with full path
-    my $file = shift (@_);       # filename without path
-    my $doc_obj = shift (@_);
-    
-    my $verbosity = $self->{'verbosity'};
-    my $outhandle = $self->{'outhandle'};
-
-    # check the filename is okay
-    return 0 if ($file eq "" || $filename eq "");
-
-    # Add the file metadata.
-    # $assoc_url will be the srcurl. Since it is URL encoded here, it will be
-    # able to cope with special characters--including spaces--in mp3 filenames.
-    # the assocfilename generated will be a URL encoded version of the utf8 filename
-    my $assoc_url = $doc_obj->get_sourcefile(); 
-    my $dst_file = $doc_obj->get_assocfile_from_sourcefile();
-
-    # Add the file as an associated file ...
-    my $section = $doc_obj->get_top_section();
-    my $mime_type = $self->{'mime_type'} || "audio/mp3";
-    my $assoc_field = $self->{'assoc_field'} || "mp3";
-    my $assoc_name = $file;
-    $assoc_name =~ s/\.mp3$//;
-
-    $doc_obj->associate_file($filename, $dst_file, $mime_type, $section);
-    $doc_obj->add_metadata ($section, $assoc_field, $assoc_name);
-    $doc_obj->add_metadata ($section, "srcurl", $assoc_url);
-
-    my $mp3_info = get_mp3info($filename);
-    my $mp3_tags = get_mp3tag($filename,0,2);
-
-    my $metadata_fields = $self->{'metadata_fields'};
-
-    if ($metadata_fields eq "*") {
-	# Locate all info and tag metadata
-
-	foreach my $ki ( keys %$mp3_info ) {
-	    my $mp3_metavalue = $mp3_info->{$ki};
-
-	    if ($mp3_metavalue !~ m/^\s*$/s) {
-		my $mp3_metaname = "ex.id3.".lc($ki);
-		$doc_obj->add_metadata ($section, $mp3_metaname, $mp3_metavalue);
-	    }
-	}
-
-	foreach my $kt ( keys %$mp3_tags ) {
-	    my $mp3_metavalue = $mp3_tags->{$kt};
-	    
-	    if ($mp3_metavalue !~ m/^\s*$/s) {
-		my $kt_len = length($kt);
-		my $kt_initial_cap = uc(substr($kt,0,1)).lc(substr($kt,1,$kt_len-1));
-		my $mp3_metaname = "ex.id3.".$kt_initial_cap;
-		
-		$doc_obj->add_metadata ($section, $mp3_metaname, $mp3_metavalue);
-	    }
-	}
-    }
-    else {
-
-	# Restrict metadata to that specifically given
-	foreach my $field (split /,/, $metadata_fields) {
-
-	    # check info
-	    if (defined $mp3_info->{$field}) {
-
-		my $mp3i_metavalue = $mp3_info->{$field};
-		
-		if ($mp3i_metavalue !~ m/^\s*$/s) {
-		    my $mp3i_metaname = "ex.id3.".lc($field);
-		    $doc_obj->add_metadata ($section, $mp3i_metaname, $mp3i_metavalue);
-		}
-	    }
-
-	    # check tags
-	    if (defined $mp3_tags->{uc($field)}) {
-
-		my $mp3t_metavalue = $mp3_tags->{uc($field)};
-		
-		if ($mp3t_metavalue !~ m/^\s*$/s) {
-		    my $mp3t_metaname = "ex.id3.".$field;
-		    
-		    $doc_obj->add_metadata ($section, $mp3t_metaname, $mp3t_metavalue);
-		}
-	    }
-	    
-	}
-    }
-
-    $doc_obj->add_metadata ($section, "FileFormat", "MP3");
-    
-    $doc_obj->add_metadata ($section, "srcicon", "_iconmp3_");
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($section, "srclink_file", $assoc_url);
-    $doc_obj->add_metadata ($section, "srclinkFile", $assoc_url);
-    my $applet_metadata = $self->{'applet_metadata'};
-    if (defined $applet_metadata && $applet_metadata ) {
-	my $applet_html 
-	    = gen_mp3applet("_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/[srcurl]");
-	$doc_obj->add_metadata ($section, "mp3applet", $applet_html);
-    }
-
-    my $assoc_images = $self->{'assoc_images'};
-    if (defined $assoc_images && $assoc_images) {
-	my @search_terms = ();
-
-	my $title  = $mp3_tags->{'TITLE'};
-	my $artist = $mp3_tags->{'ARTIST'};
-
-	if (defined $title && $title ne "") {
-
-	    push(@search_terms,$title);
-
-	    if (defined $artist && $artist ne "") {
-		push(@search_terms,$artist);
-	    }
-	}
-	else {
-	    push(@search_terms,$assoc_name);
-	}
-
-	push(@search_terms,"song");
-
-	my $output_dir = $filename;
-	$output_dir =~ s/\.\w+$//;
-
-	my ($imgref_urls) = giget(\@search_terms,$output_dir);
-
-	my $gi_base = gi_url_base();
-	my $gi_query_url = gi_query_url(\@search_terms);
-
-	$doc_obj->add_metadata ($section, "giquery", "<a href=\"$gi_base$gi_query_url\" target=giwindow>");
-	$doc_obj->add_metadata ($section, "/giquery", "</a>");
-
-	for (my $i=1; $i<=2; $i++) {
-	    my $img_filename = "$output_dir/img_$i.jpg";
-	    my $dst_file = "img_$i.jpg";
-
-	    if (-e $img_filename) {
-		$doc_obj->associate_file($img_filename, $dst_file, "image/jpeg", $section);
-
-		my $srcurl = "src=\"_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/$dst_file\"";
-
-		$doc_obj->add_metadata ($section, "img$i", 
-					"<img $srcurl>");
-		$doc_obj->add_metadata ($section, "smallimg$i", 
-					"<img $srcurl width=100>");
-
-		my $imgref_url = $imgref_urls->[$i-1];
-
-		$doc_obj->add_metadata ($section, "imgref$i", "<a href=\"$imgref_url\" target=giwindow>");
-		$doc_obj->add_metadata ($section, "/imgref$i", "</a>");
-	    }
-
-	}
-
-
-    }
-
-    return 1;
-}
-
-
-# we want to use ex.id3.Title if its there, otherwise we'll use BasePlugin method
-sub title_fallback
-{
-    my $self = shift (@_);
-    my ($doc_obj,$section,$file) = @_;
-
-    if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
-	my $mp3_title = $doc_obj->get_metadata_element ($section, "ex.id3.Title");
-	if (defined $mp3_title) {
-	    $doc_obj->add_metadata ($section, "Title", $mp3_title);
-	}
-	else {
-	    $self->BasePlugin::title_fallback($doc_obj, $section, $file);
-	}
-    }
-}
-
-
-1;
-
-
-
-
-
-
-
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MetadataCSVPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MetadataCSVPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,220 +1,0 @@
-###########################################################################
-#
-# MetadataCSVPlugin.pm -- A plugin for metadata in comma-separated value format
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 2006 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package MetadataCSVPlugin;
-
-
-use BasePlugin;
-use MetadataRead;
-
-use strict;
-no strict 'refs';
-use multiread;
-
-
-# methods with identical signatures take precedence in the order given in the ISA list.
-sub BEGIN {
-    @MetadataCSVPlugin::ISA = ('MetadataRead', 'BasePlugin');
-}
-
-
-my $arguments = [
-      { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() }
-
-];
-
-
-my $options = { 'name'     => "MetadataCSVPlugin",
-		'desc'     => "{MetadataCSVPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new
-{
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-sub get_default_process_exp
-{
-    return q^(?i)\.csv$^;
-}
-
-sub file_block_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
-	return undef; # can't recognise
-    }
-
-    # set this so we know this is a metadata file - needed for incremental 
-    # build
-    # if this file changes, then we need to reimport everything
-    $block_hash->{'metadata_files'}->{$filename_full_path} = 1;
-
-    return 1;
-}
-
-sub metadata_read
-{
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    # Read metadata from CSV files
-    my $filename = &util::filename_cat($base_dir, $file);
-    if ($filename !~ /\.csv$/ || !-f $filename) {
-	return undef;
-    }
-    print STDERR "\n<Processing n='$file' p='MetadataCSVPlugin'>\n" if ($gli);
-    print STDERR "MetadataCSVPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
-
-    my $outhandle = $self->{'outhandle'};
-    my $failhandle = $self->{'failhandle'};
-
-    # add the file to the block list so that it won't be processed in read, as we will do all we can with it here
-    &util::block_filename($block_hash,$filename);
-
-    # Read the CSV file to get the metadata
-    my $csv_file_content;
-    open(CSV_FILE, "$filename");
-    my $csv_file_reader = new multiread();
-    $csv_file_reader->set_handle('MetadataCSVPlugin::CSV_FILE');
-    $csv_file_reader->read_file(\$csv_file_content);
-    close(CSV_FILE);
-
-    # Split the file into lines and read the first line (contains the metadata names)
-    $csv_file_content =~ s/\r/\n/g;  # Handle non-Unix line endings
-    $csv_file_content =~ s/\n+/\n/g;
-    my @csv_file_lines = split(/\n/, $csv_file_content);
-    my $csv_file_field_line = shift(@csv_file_lines);
-    my @csv_file_fields = split(/\,/, $csv_file_field_line);
-    my $found_filename_field = 0;
-    for (my $i = 0; $i < scalar(@csv_file_fields); $i++) {
-	# Remove any spaces from the field names
-	$csv_file_fields[$i] =~ s/ //g;
-	if ($csv_file_fields[$i] eq "Filename") {
-	    $found_filename_field = 1;
-	}
-    }
-
-    if (!$found_filename_field) {
-	$self->print_error($outhandle, $failhandle, $gli, $filename, "No Filename field in CSV file");
-	return -1; # error
-    }
-    # Read each line of the file and assign the metadata appropriately
-    foreach my $csv_line (@csv_file_lines) {
-	# Ignore lines containing only whitespace
-	next if ($csv_line =~ /^\s*$/);
-	my $orig_csv_line = $csv_line;
-	# Build a hash of metadata name to metadata value for this line
-	my %csv_line_metadata;
-	my $i = 0;
-	$csv_line .= ",";  # To make the regular expressions simpler
-	while ($csv_line ne "") {
-	    # Metadata values containing commas are quoted
-	    if ($csv_line =~ s/^\"(.*?)\"\,//) {
-		# Only bother with non-empty values
-		if ($1 ne "" && defined($csv_file_fields[$i])) {
-		    if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
-			$csv_line_metadata{$csv_file_fields[$i]} = [];
-		    }
-		    push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $1);
-		}
-	    }
-	    # Normal comma-separated case
-	    elsif ($csv_line =~ s/^(.*?)\,//) {
-		# Only bother with non-empty values
-		if ($1 ne "" && defined($csv_file_fields[$i])) {
-		    if (!defined $csv_line_metadata{$csv_file_fields[$i]}) {
-			$csv_line_metadata{$csv_file_fields[$i]} = [];
-		    }
-		    push (@{$csv_line_metadata{$csv_file_fields[$i]}}, $1);
-		}
-	    }
-	    # The line must be formatted incorrectly
-	    else {
-		$self->print_error($outhandle, $failhandle, $gli, $filename, "Badly formatted CSV line: $csv_line");
-		last;
-	    }
-
-	    $i++;
-	}
-
-	# We can't associate any metadata without knowing the file to associate it with
-	my $csv_line_filename_array = $csv_line_metadata{"Filename"};
-	if (!defined $csv_line_filename_array) {
-	    $self->print_error($outhandle, $failhandle, $gli, $filename, "No Filename metadata in CSV line: $orig_csv_line");
-	    next;
-	}
-	my $csv_line_filename = shift(@$csv_line_filename_array);
-	delete $csv_line_metadata{"Filename"};
-
-
- 	# Associate the metadata now
-	$csv_line_filename = &util::filename_to_regex($csv_line_filename);
-
- 	$extrametadata->{$csv_line_filename} = \%csv_line_metadata;
- 	push(@$extrametakeys, $csv_line_filename);
-	# record which file the metadata came from 
-	if (!defined $extrametafile->{$csv_line_filename}) {
-	    $extrametafile->{$csv_line_filename} = {};
-	}
-	# maps the file to full path
-	$extrametafile->{$csv_line_filename}->{$file} = $filename;
-    }
-}
-
-sub print_error
-{
-
-    my $self = shift(@_);
-    my ($outhandle, $failhandle, $gli, $file, $error) = @_;
-
-    print $outhandle "MetadataCSVPlugin Error: $file: $error\n";
-    print $failhandle "MetadataCSVPlugin Error: $file: $error\n";
-    print STDERR "<ProcessingError n='$file' r='$error'/>\n" if ($gli);
-}
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/PDFPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/PDFPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,435 +1,0 @@
-###########################################################################
-#
-# PDFPlugin.pm -- reasonably with-it pdf plugin
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999-2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package PDFPlugin;
-
-use strict;
-no strict 'refs'; # so we can use a var for filehandles (e.g. STDERR)
-
-use ReadTextFile;
-use unicode;
-
-use AutoLoadConverters;
-use ConvertBinaryFile;
-
-@PDFPlugin::ISA = ('ConvertBinaryFile', 'AutoLoadConverters', 'ReadTextFile');
-
-
-my $convert_to_list =
-    [ {	'name' => "auto",
-	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-      {	'name' => "html",
-	'desc' => "{ConvertBinaryFile.convert_to.html}" },
-      {	'name' => "text",
-	'desc' => "{ConvertBinaryFile.convert_to.text}" },
-      { 'name' => "pagedimg_jpg",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_jpg}"},
-      { 'name' => "pagedimg_gif",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_gif}"},
-      { 'name' => "pagedimg_png",
-	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_png}"}, 
-      ];
-
-
-my $arguments = 
-    [
-     { 'name' => "convert_to",
-       'desc' => "{ConvertBinaryFile.convert_to}",
-       'type' => "enum",
-       'reqd' => "yes",
-       'list' => $convert_to_list, 
-       'deft' => "html" },	 
-     { 'name' => "process_exp",
-       'desc' => "{BasePlugin.process_exp}",
-       'type' => "regexp",
-       'deft' => &get_default_process_exp(),
-       'reqd' => "no" },
-     { 'name' => "block_exp",
-       'desc' => "{BasePlugin.block_exp}",
-       'type' => "regexp",
-       'deft' => &get_default_block_exp() },
-     { 'name' => "metadata_fields",
-       'desc' => "{HTMLPlugin.metadata_fields}",
-       'type' => "string",
-       'deft' => "Title,Author,Subject,Keywords" },
-      { 'name' => "metadata_field_separator",
-	'desc' => "{HTMLPlugin.metadata_field_separator}",
-	'type' => "string",
-	'deft' => "" },
-     { 'name' => "noimages",
-       'desc' => "{PDFPlugin.noimages}",
-       'type' => "flag" },
-     { 'name' => "allowimagesonly",
-       'desc' => "{PDFPlugin.allowimagesonly}",
-       'type' => "flag" },
-     { 'name' => "complex",
-       'desc' => "{PDFPlugin.complex}",
-       'type' => "flag" },
-     { 'name' => "nohidden",
-       'desc' => "{PDFPlugin.nohidden}",
-       'type' => "flag" },
-     { 'name' => "zoom",
-       'desc' => "{PDFPlugin.zoom}",
-       'deft' => "2",
-       'range' => "1,3", # actually the range is 0.5-3 
-       'type' => "int" },
-     { 'name' => "use_sections",
-       'desc' => "{PDFPlugin.use_sections}",
-       'type' => "flag" },
-     { 'name' => "description_tags",
-       'desc' => "{HTMLPlugin.description_tags}",
-       'type' => "flag" }
-     ];
-
-my $options = { 'name'     => "PDFPlugin",
-		'desc'     => "{PDFPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in PDF can be replaced with GS-generated html		
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@$inputargs,"-title_sub");
-    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $auto_converter_self = new AutoLoadConverters($pluginlist,$inputargs,$hashArgOptLists,["PDFBoxConverter"],1);
-    my $cbf_self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-    my $self = BasePlugin::merge_inheritance($auto_converter_self, $cbf_self);
-    
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-    
-    $self = bless $self, $class;
-    $self->{'file_type'} = "PDF";
-
-    # these are passed through to gsConvert.pl by ConvertBinaryFile.pm
-    my $zoom = $self->{"zoom"};
-    $self->{'convert_options'} = "-pdf_zoom $zoom";
-    $self->{'convert_options'} .= " -pdf_complex" if $self->{"complex"};
-    $self->{'convert_options'} .= " -pdf_nohidden" if $self->{"nohidden"};
-    $self->{'convert_options'} .= " -pdf_ignore_images" if $self->{"noimages"};
-    $self->{'convert_options'} .= " -pdf_allow_images_only" if $self->{"allowimagesonly"};
-
-    # check convert_to
-    if ($self->{'convert_to'} eq "text" && $ENV{'GSDLOS'} =~ /^windows$/i) {
-	print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
-	$self->{'convert_to'} = "html";
-    }
-    elsif ($self->{'convert_to'} eq "auto") {
-	# choose html ?? is this the best option
-	$self->{'convert_to'} = "html";
-    }
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
-
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    # following title_sub removes "Page 1" added by pdftohtml, and a leading
-    # "1", which is often the page number at the top of the page. Bad Luck
-    # if your document title actually starts with "1 " - is there a better way?
-    push(@$specific_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
-    my $associate_tail_re = $self->{'associate_tail_re'};
-    if ((defined $associate_tail_re) && ($associate_tail_re ne "")) {
-	push(@$specific_options, "-associate_tail_re", $associate_tail_re);
-    }
-    push(@$specific_options, "-file_rename_method", "none");
-    
-    if ($secondary_plugin_name eq "HTMLPlugin") {
-	# pdftohtml always produces utf8 - What about pdfbox???
-	# push(@$specific_options, "-input_encoding", "utf8");
-	push(@$specific_options, "-extract_language") if $self->{'extract_language'};
-	push(@$specific_options, "-processing_tmp_files");
-	# Instruct HTMLPlug (when eventually accessed through read_into_doc_obj) 
-	# to extract these metadata fields from the HEAD META fields
-	if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'} =~ /\S/) {
-	    push(@$specific_options,"-metadata_fields",$self->{'metadata_fields'});
-	} else {
-	    push(@$specific_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
-	}
-	if (defined $self->{'metadata_field_separator'} && $self->{'metadata_field_separator'} =~ /\S/) {
-	    push(@$specific_options,"-metadata_field_separator",$self->{'metadata_field_separator'});
-	}
-	if ($self->{'use_sections'} || $self->{'description_tags'}) {
-	    $self->{'description_tags'} = 1;
-	    push(@$specific_options, "-description_tags");
-	}
-    }
-    elsif ($secondary_plugin_name eq "PagedImagePlugin") {
-	push(@$specific_options, "-screenviewsize", "1000");
-	push(@$specific_options, "-enable_cache");
-	push(@$specific_options, "-processing_tmp_files");
-    }
-
-    $self = bless $self, $class;
-    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
-    return $self;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.pdf$^;
-}
-
-# so we don't inherit HTMLPlug's block exp...
-sub get_default_block_exp {
-    return "";
-}
-
-sub init {
-    my $self = shift (@_);
-
-    # ConvertBinaryFile init
-    $self->SUPER::init(@_);
-    $self->AutoLoadConverters::init(@_);
-
-}
-
-sub begin {
-    my $self = shift (@_);
-
-    $self->AutoLoadConverters::begin(@_);
-    $self->SUPER::begin(@_);
-
-}
-
-sub deinit {
-    my $self = shift (@_);
-    
-    $self->AutoLoadConverters::deinit(@_);
-    $self->SUPER::deinit(@_);
-
-}
-  
-# By setting hashing to be on ga xml this ensures that two 
-# PDF files that are identical except for the metadata
-# to hash to different values. Without this, when each PDF
-# file is converted to HTML there is a chance that they 
-# will both be *identical* if the conversion utility does
-# not embed the metadata in the generated HTML. This is
-# certainly the case when PDFBOX is being used. 
-
-# This change makes this convert to based plugin more 
-# consistent with the original vision that the same document 
-# with different metadata should
-# be seen as different.
-
-sub get_oid_hash_type {
-    my $self = shift (@_);
-    return "hash_on_ga_xml";
-}
-  
-  
-sub tmp_area_convert_file {
-
-    my $self = shift (@_);
-    return $self->AutoLoadConverters::tmp_area_convert_file(@_);
-
-}
-
-sub convert_post_process
-{
-    my $self = shift (@_);
-    my ($conv_filename) = @_;
-
-    my $outhandle=$self->{'outhandle'};
-
-    #$self->{'input_encoding'} = "utf8"; # The output is always in utf8 (is it?? it is for html, but what about other types?)
-    #my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
-
-    # read in file ($text will be in utf8)
-    my $text = "";
-    # encoding will be utf8 for html files - what about other types? will we do this step for them anyway?
-    $self->read_file ($conv_filename, "utf8", "", \$text);
-
-    # To support the use_sections option with PDFBox: Greenstone splits PDFs into pages for 
-    # sections. The PDFPlugin code wants each new page to be prefixed with <a name=pagenum></a>,
-    # which it then splits on to generate page-based sections. However, that's not what PDFBox 
-    # generates in its HTML output. Fortunately, PDFBox does have its own page-separator: it
-    # embeds each page in an extra div. The div opener is: 
-    # <div style=\"page-break-before:always; page-break-after:always\">
-    # The PDFPlugin now looks for this and prefixes <a name=0></a> to each such div. (The 
-    # pagenumber is fixed at 0 since I'm unable to work out how to increment the pagenum during 
-    # a regex substitution even with regex extensions on.) Later, when we process each section 
-    # to get the pagenum, PDFBox's output for this is pre-processed by having a loopcounter 
-    # that increments the pagenum for each subsequent section.
-
-    #$pdfbox_pageheader="\<div style=\"page-break-before:always; page-break-after:always\">";
-    my $loopcounter = 0; # used later on!
-    $text =~ s@\<div style=\"page-break-before:always; page-break-after:always\">@<a name=$loopcounter></a><div style=\"page-break-before:always; page-break-after:always\">@g;
-
-
-    # Calculate number of pages based on <a ...> tags (we have a <a name=1> etc
-    # for each page).  Metadata based on this calculation not set until process()
-    # 
-    # Note: this is done even if we are not breaking the document into pages as it might
-    # be useful to give an indication of document length in browser through setting
-    # num_pages as metadata.
-    my @pages = ($text =~ m/\<[Aa] name=\"?\w+\"?>/ig); #<div style=\"?page-break-before:always; page-break-after:always\"?>
-    my $num_pages = scalar(@pages);
-    $self->{'num_pages'} = $num_pages;
-
-    if ($self->{'use_sections'}
-	&& $self->{'converted_to'} eq "HTML") {
-
-	print $outhandle "PDFPlugin: Calculating sections...\n";
-
-	# we have "<a name=1></a>" etc for each page
-	# it may be <A name=
-	my @sections = split('<[Aa] name=', $text);
-
-	my $top_section = "";
-
-	if (scalar (@sections) == 1) { #only one section - no split!
-	    print $outhandle "PDFPlugin: warning - no sections found\n";
-	} else {
-	    $top_section .= shift @sections; # keep HTML header etc as top_section
-	}
-
-	# handle first section specially for title? Or all use first 100...
-	
-	my $title = $sections[0];
-	$title =~ s/^\"?\w+\"?>//; # specific for pdftohtml...
-	$title =~ s/<\/([^>]+)><\1>//g; # (eg) </b><b> - no space
-	$title =~ s/<[^>]*>/ /g;
-	$title =~ s/(?:&nbsp;|\xc2\xa0)/ /g; # utf-8 for nbsp...
-	$title =~ s/^\s+//s;
-	$title =~ s/\s+$//;
-	$title =~ s/\s+/ /gs;
-	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
-	$title =~ s/^\s+//s; # in case title_sub introduced any...
-	$title = substr ($title, 0, 100);
-	$title =~ s/\s\S*$/.../;
-
-
-	if (scalar (@sections) == 1) { # no sections found
-	    $top_section .= $sections[0];
-	    @sections=();
-	} else {
-	    $top_section .= "<!--<Section>\n<Metadata name=\"Title\">$title</Metadata>\n-->\n <!--</Section>-->\n";
-	}
-
-	# add metadata per section...
-	foreach my $section (@sections) {
-	    # section names are not always just digits, may be like "outline"
-	    $section =~ s@^\"?(\w+)\"?></a>@@; # leftover from split expression...
-
-	    $title = $1; # Greenstone does magic if sections are titled digits
-
-	    # A title of pagenum=0 means use_sections is being applied on output from PDFBox,
-	    # which didn't originally have a <a name=incremented pagenumber></a> to split each page. 
-	    # Our Perl code then prefixed <a name=0></a> to it. Now need to increment the pagenum here:
-	    if($loopcounter > 0 || ($title eq 0 && $loopcounter == 0)) { # implies use_sections with PDFBox
-		$title = ++$loopcounter;
-	    }
-
-	    if (! defined($title) ) {
-		print STDERR "no title: $section\n";
-		$title = " "; # get rid of the undefined warning in next line
-	    }
-	    my $newsection = "<!-- from PDFPlugin -->\n<!-- <Section>\n";
-	    $newsection .= "<Metadata name=\"Title\">" . $title
-		. "</Metadata>\n--><br />\n";
-	    $newsection .= $section;
-	    $newsection .= "<!--</Section>-->\n";
-	    $section = $newsection;
-	}
-
-	$text=join('', ($top_section, @sections));
-    }
-
-    if ($self->{'use_sections'}
-	&& $self->{'converted_to'} eq "text") {
-	print STDERR "**** When converting PDF to text, cannot apply use_sections\n";
-    }
-
-
-    # The following should no longer be needed, now that strings
-    # read in are Unicode aware (in the Perl sense) rather than
-    # raw binary strings that just happen to be UTF-8 compliant
-
-    # turn any high bytes that aren't valid utf-8 into utf-8.
-##    unicode::ensure_utf8(\$text);
-
-    # Write it out again!
-    $self->utf8_write_file (\$text, $conv_filename);
-}
-
-
-# do plugin specific processing of doc_obj for HTML type
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my $result = $self->process_type($base_dir,$file,$doc_obj);
-
-    # fix up the extracted date metadata to be in Greenstone date format,
-    # and fix the capitalisation of 'date'
-    my $cursection = $doc_obj->get_top_section();
-    foreach my $datemeta (@{$doc_obj->get_metadata($cursection, "date")}) {
-	$doc_obj->delete_metadata($cursection, "date", $datemeta);
-
-	# We're just interested in the date bit, not the time
-	# some pdf creators (eg "Acrobat 5.0 Scan Plug-in for Windows")
-	# set a /CreationDate, and set /ModDate to 000000000. pdftohtml
-	# extracts the ModDate, so it is 0...
-	$datemeta =~ /(\d+)-(\d+)-(\d+)/;
-	my ($year, $month, $day) = ($1,$2,$3);
-	if (defined($year) && defined($month) && defined($day)) {
-	    if ($year == 0) {next}
-	    if ($year < 100) {$year += 1900} # just to be safe
-	    if ($month =~ /^\d$/) {$month="0$month"} # single digit
-	    if ($day =~ /^\d$/) {$day="0$day"} # single digit
-	    my $date="$year$month$day";
-	    $doc_obj->add_utf8_metadata($cursection, "Date", $date);
-	}
-    }
-
-    $doc_obj->add_utf8_metadata($cursection, "NumPages", $self->{'num_pages'}) if defined $self->{'num_pages'};
-    
-    if ($self->{'use_sections'} && $self->{'converted_to'} eq "HTML") {
-	# we explicitly make it a paged document, cos greenstone won't get it
-	# right if any section has an empty title, or one with letters in it
-	$doc_obj->set_utf8_metadata_element ($cursection, "gsdlthistype", "Paged");
-    }
-
-    return $result;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/PrintInfo.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/PrintInfo.pm	(revision 26943)
+++ 	(revision )
@@ -1,288 +1,0 @@
-###########################################################################
-#
-# PrintInfo - most base plugin
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-## Most basic plugin, just handles parsing the arguments and printing out descriptions. Used for plugins and Extractor plugins
-
-package PrintInfo;
-
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-}
-
-eval {require bytes};
-eval "require diagnostics"; # some perl distros (eg mac) don't have this
-
-# suppress the annoying "subroutine redefined" warning that various
-# plugins cause under perl 5.6
-$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
-
-use strict;
-no strict 'subs';
-
-use gsprintf 'gsprintf';
-use parse2;
-use printusage;
-
-my $arguments = [];
-
-my $options = { 'name'     => "PrintInfo",
-		'desc'     => "{PrintInfo.desc}",
-		'abstract' => "yes",
-		'inherits' => "no",
-		'args'     => $arguments };
-
-# $auxiliary_plugin argument passed in by "on-the-side" plugin helpers such as Extractors and ImageConverter. We don't want parsing of args done by them.
-sub new 
-{
-    my $class = shift (@_);
-    my ($pluginlist,$args,$hashArgOptLists, $auxiliary) = @_;
-    my $plugin_name = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
-
-    if ($plugin_name eq $class) {
-	push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-	push(@{$hashArgOptLists->{"OptList"}},$options);
-    }
-    my $self = {};
-    $self->{'outhandle'} = STDERR;
-    $self->{'option_list'} = $hashArgOptLists->{"OptList"};
-    $self->{"info_only"} = 0;
-
-    # Check if gsdlinfo is in the argument list or not - if it is, don't parse 
-    # the args, just return the object.  
-    foreach my $strArg (@{$args})
-    {
-	if($strArg eq "-gsdlinfo")
-	{
-	    $self->{"info_only"} = 1;
-	    return bless $self, $class;
-	}
-    }
-    
-    if (defined $auxiliary) { # don't parse the args here
-	return bless $self, $class;
-    }
-
-    if(parse2::parse($args,$hashArgOptLists->{"ArgList"},$self) == -1)
-    {
-	my $classTempClass = bless $self, $class;
-	print STDERR "<BadPlugin p=$plugin_name>\n";
-	&gsprintf(STDERR, "\n{PrintInfo.bad_general_option}\n", $plugin_name);
-	$classTempClass->print_txt_usage("");  # Use default resource bundle
-	die "\n";
-    }
-
-    delete $self->{"info_only"};
-    # else parsing was successful.
-
-    $self->{'plugin_type'} = $plugin_name;
- 
-    return bless $self, $class;
-    
-}   
-
-#sub init {
-#}
-
-sub set_incremental {
-    my $self = shift(@_);
-    my ($incremental_mode) = @_;
-
-    if (!defined $incremental_mode) {
-	$self->{'incremental'} = 0;
-	$self->{'incremental_mode'} = "none";
-    }
-    elsif ($incremental_mode eq "all") {
-	$self->{'incremental'} = 1;
-	$self->{'incremental_mode'} = "all";
-    }
-    else {
-	# none, onlyadd
-	$self->{'incremental'} = 0;
-	$self->{'incremental_mode'} = $incremental_mode;
-    }	
-}
-
-sub get_arguments
-{
-    my $self = shift(@_);
-    my $optionlistref = $self->{'option_list'};
-    my @optionlist = @$optionlistref;
-    my $pluginoptions = pop(@$optionlistref);
-    my $pluginarguments = $pluginoptions->{'args'};
-    return $pluginarguments;
-}
-
-
-sub print_xml_usage
-{
-    my $self = shift(@_);
-    my $header = shift(@_);
-    my $high_level_information_only = shift(@_);
-    
-    # XML output is always in UTF-8
-    gsprintf::output_strings_in_UTF8;
-
-    if ($header) {
-	&PrintUsage::print_xml_header("plugin");
-    }
-    $self->print_xml($high_level_information_only);
-}
-
-
-sub print_xml
-{
-    my $self = shift(@_);
-    my $high_level_information_only = shift(@_);
-
-    my $optionlistref = $self->{'option_list'};
-    my @optionlist = @$optionlistref;
-    my $pluginoptions = shift(@$optionlistref);
-    return if (!defined($pluginoptions));
-
-    # Find the process and block default expressions in the plugin arguments
-    my $process_exp = "";
-    my $block_exp = "";
-    if (defined($pluginoptions->{'args'})) {
-	foreach my $option (@{$pluginoptions->{'args'}}) {
-	    if ($option->{'name'} eq "process_exp") {
-		$process_exp = $option->{'deft'};
-	    }
-	    if ($option->{'name'} eq "block_exp") {
-		$block_exp = $option->{'deft'};
-	    }
-	}
-    }
-
-    gsprintf(STDERR, "<PlugInfo>\n");
-    gsprintf(STDERR, "  <Name>$pluginoptions->{'name'}</Name>\n");
-    my $desc = gsprintf::lookup_string($pluginoptions->{'desc'});
-    $desc =~ s/</&amp;lt;/g; # doubly escaped
-    $desc =~ s/>/&amp;gt;/g;
-    gsprintf(STDERR, "  <Desc>$desc</Desc>\n");
-    gsprintf(STDERR, "  <Abstract>$pluginoptions->{'abstract'}</Abstract>\n");
-    gsprintf(STDERR, "  <Inherits>$pluginoptions->{'inherits'}</Inherits>\n");
-    gsprintf(STDERR, "  <Processes>$process_exp</Processes>\n");
-    gsprintf(STDERR, "  <Blocks>$block_exp</Blocks>\n");
-    gsprintf(STDERR, "  <Explodes>" . ($pluginoptions->{'explodes'} || "no") . "</Explodes>\n");
-    # adding new option that works with replace_srcdoc_with_html.pl
-    gsprintf(STDERR, "  <SourceReplaceable>" . ($pluginoptions->{'srcreplaceable'} || "no") . "</SourceReplaceable>\n");
-    unless (defined($high_level_information_only)) {
-	gsprintf(STDERR, "  <Arguments>\n");
-	if (defined($pluginoptions->{'args'})) {
-	    &PrintUsage::print_options_xml($pluginoptions->{'args'});
-	}
-	gsprintf(STDERR, "  </Arguments>\n");
-
-	# Recurse up the plugin hierarchy
-	$self->print_xml();
-    }
-    gsprintf(STDERR, "</PlugInfo>\n");
-}
-
-
-sub print_txt_usage
-{
-    my $self = shift(@_);
-    # Print the usage message for a plugin (recursively)
-    my $descoffset = $self->determine_description_offset(0);
-    $self->print_plugin_usage($descoffset, 1);
-}
-
-
-sub determine_description_offset
-{
-    my $self = shift(@_);
-    my $maxoffset = shift(@_);
-
-    my $optionlistref = $self->{'option_list'};
-    my @optionlist = @$optionlistref;
-    my $pluginoptions = shift(@$optionlistref);
-    return $maxoffset if (!defined($pluginoptions));
-
-    # Find the length of the longest option string of this plugin
-    my $pluginargs = $pluginoptions->{'args'};
-    if (defined($pluginargs)) {
-	my $longest = &PrintUsage::find_longest_option_string($pluginargs);
-	if ($longest > $maxoffset) {
-	    $maxoffset = $longest;
-	}
-    }
-
-    # Recurse up the plugin hierarchy
-    $maxoffset = $self->determine_description_offset($maxoffset);
-    $self->{'option_list'} = \@optionlist;
-    return $maxoffset;
-}
-
-
-sub print_plugin_usage
-{
-    my $self = shift(@_);
-    my $descoffset = shift(@_);
-    my $isleafclass = shift(@_);
-
-    my $optionlistref = $self->{'option_list'};
-    my @optionlist = @$optionlistref;
-    my $pluginoptions = shift(@$optionlistref);
-    return if (!defined($pluginoptions));
-
-    my $pluginname = $pluginoptions->{'name'};
-    my $pluginargs = $pluginoptions->{'args'};
-    my $plugindesc = $pluginoptions->{'desc'};
-
-    # Produce the usage information using the data structure above
-    if ($isleafclass) {
-	if (defined($plugindesc)) {
-	    gsprintf(STDERR, "$plugindesc\n\n");
-	}
-	gsprintf(STDERR, " {common.usage}: plugin $pluginname [{common.options}]\n\n");
-    }
-
-    # Display the plugin options, if there are some
-    if (defined($pluginargs)) {
-	# Calculate the column offset of the option descriptions
-	my $optiondescoffset = $descoffset + 2;  # 2 spaces between options & descriptions
-
-	if ($isleafclass) {
-	    gsprintf(STDERR, " {common.specific_options}:\n");
-	}
-	else {
-	    gsprintf(STDERR, " {common.general_options}:\n", $pluginname);
-	}
-
-	# Display the plugin options
-	&PrintUsage::print_options_txt($pluginargs, $optiondescoffset);
-    }
-
-    # Recurse up the plugin hierarchy
-    $self->print_plugin_usage($descoffset, 0);
-    $self->{'option_list'} = \@optionlist;
-}
-
-
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ProCitePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ProCitePlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,253 +1,0 @@
-###########################################################################
-#
-# ProCitePlugin.pm -- A plugin for (exported) ProCite databases
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 1999-2004 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ProCitePlugin;
-
-
-use multiread;
-use SplitTextFile;
-use MetadataRead;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-# ProCitePlugin is a sub-class of SplitTextFile
-sub BEGIN {
-    @ProCitePlugin::ISA = ('MetadataRead', 'SplitTextFile');
-}
-
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "split_exp",
-	'desc' => "{SplitTextFile.split_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_split_exp(),
-	'reqd' => "no" },
-      
-      # The interesting options
-      { 'name' => "entry_separator",
-	'desc' => "{ProCitePlugin.entry_separator}",
-	'type' => "string",
-	'reqd' => "no",
-	'deft' => "//" },
-      ];
-
-my $options = { 'name'     => "ProCitePlugin",
-		'desc'     => "{ProCitePlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-
-# This plugin processes exported ProCite files with the suffix ".txt"
-sub get_default_process_exp
-{
-    return q^(?i)(\.txt)$^;
-}
-
-
-# This plugin splits the input text at every line
-sub get_default_split_exp
-{
-    return q^\n^;
-}
-
-
-sub new
-{
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-my %crazy_workform_mapping =
-    ( "A", "Book, Long Form",
-      "B", "Book, Short Form",
-      "C", "Journal, Long Form",
-      "D", "Journal, Short Form",
-      "E", "Report",
-      "F", "Newspaper",
-      "G", "Dissertation",
-      "H", "Trade Catalog",
-      "I", "Letter (Correspondence)",
-      "J", "Manuscript",
-      "K", "Conference Proceedings",
-      "L", "Map",
-      "M", "Music Score",
-      "N", "Sound Recording",
-      "O", "Motion Picture",
-      "P", "Audiovisual Material",
-      "Q", "Video Recording",
-      "R", "Art Work",
-      "S", "Computer Program",
-      "T", "Data File" );
-
-
-sub read_file
-{
-    my $self = shift (@_);
-    my ($filename, $encoding, $language, $textref) = @_;
-    
-    # Store the workform definitions for this file
-    my %workform_definitions = ();
-
-    # Read the contents of the file into $textref
-    open(PROCITE_FILE, "<$filename");
-    my $reader = new multiread();
-    $reader->set_handle ('ProCitePlugin::PROCITE_FILE');
-    $reader->set_encoding ($encoding);
-    $reader->read_file ($textref);
-    close(PROCITE_FILE);
-
-    # Read the workform definitions at the start of the file
-    while ($$textref =~ /^\<Workform Definition\>/) {
-	# Remove the workform definition line so it is not processed later as a record
-	$$textref =~ s/^\<Workform Definition\>(.*)\n//;
-	my $workform_definition = $1;
-	# Parse the workform definitions and store them for later
-	$workform_definition =~ s/^\"([^\"]*)\",//;
-	my $workform_name = $1;
-	my @workform_values;
-	while ($workform_definition !~ /^\s*$/) {
-	    $workform_definition =~ s/^\"([^\"]*)\",?//;
-	    my $workform_field = $1;
-	    push(@workform_values, $workform_field);
-	}
-	
-	# Remember this workform definition for when we're reading the records
-	$workform_definitions{$workform_name} = \@workform_values;
-    }
-
-    $self->{'workform_definitions'}->{$filename} = \%workform_definitions;
-}
-
-
-sub process
-{
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $filename = &util::filename_cat($base_dir, $file);
-    my $cursection = $doc_obj->get_top_section();
-
-    # Build up an HTML view of the record for easy display at run-time
-    my $html_record = "<table>";
-
-    # Read the record's workform indicator and record number
-    #$$textref =~ s/^\"([^\"]*)\",\"([^\"]*)\",//;
-    $$textref =~  s/^\"([^\"]*)\",//;
-    my $workform_indicator = $1;
-
-    # some procite files have a record number next
-    
-    my $recordnum = $$textref =~ s/^\"(\d*)\",//;
-    $recordnum = "undefined" unless defined $recordnum;
-
-    # If necessary, map the workform indicator into something useful
-    if ($crazy_workform_mapping{$workform_indicator}) {
-	$workform_indicator = $crazy_workform_mapping{$workform_indicator};
-    }
-
-    # Check we know about the workform of this record
-    my %workform_definitions = %{$self->{'workform_definitions'}->{$filename}};
-    if (!$workform_definitions{$workform_indicator}) {
-	print STDERR "Unknown workform $workform_indicator!\n";
-	return 0;
-    }
-
-    # Store the full record as the document text
-    $doc_obj->add_utf8_text($cursection, $$textref);
-
-    # Store workform and record number as metadata
-    $doc_obj->add_utf8_metadata($cursection, "Workform", $workform_indicator);
-    $doc_obj->add_utf8_metadata($cursection, "RecordNumber", $recordnum);
-    
-    # Store FileFormat metadata
-    $doc_obj->add_metadata($cursection, "FileFormat", "ProCite");
-
-    $html_record .= "<tr><td valign=top><b>Record Number: </b></td><td valign=top>$recordnum</td></tr>";
-
-    my @workform_values = @{$workform_definitions{$workform_indicator}};
-
-    # Read each field (surrounded by quotes) of the record
-    my $fieldnum = 0;
-    while ($$textref !~ /^\s*$/) {
-	$$textref =~ s/^\"([^\"]*)\",?//;
-	my $field_value_raw = $1;
-
-	# Add non-empty metadata values to the document
-	unless ($field_value_raw eq "") {
-	    # Add the display name of the metadata field for format statement convenience
-	    my $field_name = $workform_values[$fieldnum];
-	    #unless ($field_name eq "---") {
-	#	my $meta_name = "Field" . ($fieldnum + 1) . "Name";
-	#	$doc_obj->add_utf8_metadata($cursection, $meta_name, $field_name);
-	 #   }
-	    if ($field_name eq "---") {
-		$field_name = "Field" . ($fieldnum + 1);
-	    }
-	    $html_record .= "<tr><td valign=top><b>$field_name: </b></td><td valign=top>";
-
-	    # Multiple metadata values are separated with "//"
-	    #foreach my $field_value (split(/\/\//, $field_value_raw)) {
-	    foreach my $field_value (split($self->{'entry_separator'}, $field_value_raw)) {
-		#my $meta_name = "Field" . ($fieldnum + 1) . "Value";
-		#$doc_obj->add_utf8_metadata($cursection, $meta_name, $field_value);
-		$doc_obj->add_utf8_metadata($cursection, $field_name, $field_value);
-		$html_record .= $field_value . "<br>";
-	    }
-
-	    $html_record .= "</td></tr>";
-	}
-
-	$fieldnum++;
-    }
-
-    $html_record .= "</table>";
-    # Store HTML view of record as metadata
-    $doc_obj->add_utf8_metadata($cursection, "HTMLDisplay", $html_record);
-
-    # Record was processed successfully
-    return 1;
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/RogPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/RogPlugin.pm	(revision 26943)
+++ 	(revision )
@@ -1,283 +1,0 @@
-###########################################################################
-#
-# RogPlugin.pm -- simple text plugin
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# creates simple single-level document from .rog or .mdb files 
-
-package RogPlugin;
-
-use BasePlugin;
-use MetadataRead;
-use sorttools;
-use doc; 
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @RogPlugin::ISA = ('MetadataRead', 'BasePlugin');
-}
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      ];
-
-my $options = { 'name'     => "RogPlugin",
-		'desc'     => "{RogPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args' => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-
-# This plugin processes files with the suffix ".mdb" or ".rog"
-sub get_default_process_exp {
-    return q^(?i)\.(mdb|rog)$^;
-}
-
-sub read_rog_record
-{
-    my ($self,$file_buffer, $seclevel) = @_;
-
-    my $next_line = $file_buffer->{'next_line'};
-
-    return 0 if (!defined $next_line);
-
-    if ($next_line eq "")
-    {
-	my $line;
-	while(defined($line=<FILE>))  
-	{
-	    $line =~ s/\r$//;
-	    $file_buffer->{'line_no'}++;
-	    next if ($line =~ m/^\#/);
-	    $next_line = $line;
-	    last;
-	}
-    }
-	    
-    if ($next_line !~ m/^song( +)\"([^\"]*)\"( +)\"([^\"]*)\"( +)(\d+)( *)$/) 
-    {
-	print STDERR "Error: Malformed Rog file: $next_line";
-	return 0;
-    }
-    else
-    {
-	# init default values
-	$file_buffer->{'song'}->{'tempo'}    = 120;
-	$file_buffer->{'song'}->{'ks_type'}  = 0;
-	$file_buffer->{'song'}->{'ks_num'}   = 0;
-	$file_buffer->{'song'}->{'metadata'} = [];
-	$file_buffer->{'song'}->{'content'}  = "";
-	
-	$file_buffer->{'song'}->{'subcol'} = $2;
-	$file_buffer->{'song'}->{'title'}  = $4;
-	$file_buffer->{'song'}->{'tval'}   = $6;
-
-	chomp($next_line);
-	my $content = $next_line;
-	if (defined $seclevel) 
-	{
-	    $content.= " $seclevel";
-	}
-	$content .= "\n";
-
-	$file_buffer->{'song'}->{'content'} = $content;
-
-
-	my $line;
-	while(defined($line=<FILE>))  
-	{
-	    $line =~ s/\r$//;
-
-	    $file_buffer->{'line_no'}++;
-	    next if ($line =~ m/^\#/);
-	
-	    if ($line =~ m/^song/) 
-	    {	
-		$file_buffer->{'next_line'} = $line;
-		return 1;
-	    }
-	    elsif ($line =~ m/^tempo( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'tempo'} = $2;
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    } 
-	    elsif ($line =~ m/^keysig( +)(\d+)( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'ks_type'} = $2;
-		$file_buffer->{'song'}->{'ks_num'}  = $4;
-		$file_buffer->{'song'}->{'content'} .= $line;	   
-	    } 
-	    elsif ($line =~ m/^timesig( +)(\d+)( +)(\d+)( *)$/) 
-	    {
-		$file_buffer->{'song'}->{'ts_numer'} = $2;
-		$file_buffer->{'song'}->{'ts_denom'} = $4;
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	    elsif ($line =~ m/^metadata ([^:]*): (.*)/)
-	    {
-		push(@{$file_buffer->{'song'}->{'metadata'}},[$1,$2]);
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	    else
-	    {
-		$file_buffer->{'song'}->{'content'} .= $line;
-	    }
-	}
-	
-	$file_buffer->{'next_line'} = undef;
-    }
-
-    return 1;
-}
-
-sub process_rog_record
-{
-    my ($self,$file,$metadata,$song,$processor) = @_;
-
-    # create a new document
-    my $doc_obj = new doc ($file, "indexed_doc", $self->{'file_rename_method'});
-    my $cursection = $doc_obj->get_top_section();
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-
-    my $title = $song->{'title'};
-    my $title_safe = $title;
-    $title_safe =~ s/\'/\\\\&apos;/g;
-
-    # add metadata 
-    $doc_obj->add_metadata($cursection, "Tempo",         $song->{'tempo'}); 
-    $doc_obj->add_metadata($cursection, "KeySigType",    $song->{'ks_type'}); 
-    $doc_obj->add_metadata($cursection, "KeySigNum",     $song->{'ks_num'}); 
-    $doc_obj->add_metadata($cursection, "SubCollection", $song->{'subcol'}); 
-    $doc_obj->add_metadata($cursection, "Title",         $title); 
-    $doc_obj->add_metadata($cursection, "TitleSafe",     $title_safe); 
-    $doc_obj->add_metadata($cursection, "TVal",          $song->{'tval'}); 
-    $doc_obj->add_metadata($cursection, "FileFormat",    "Rog");
-    $doc_obj->add_metadata($cursection, "FileSize",      (-s $file));
-
-    foreach my $md ( @{$song->{'metadata'}} )
-    {
-	$doc_obj->add_metadata($cursection, $md->[0], $md->[1]); 
-    }
-
-    # add contents as text
-    $doc_obj->add_text($cursection,$song->{'content'});
-
-    $self->extra_metadata($doc_obj,$cursection, $metadata);
-    
-    # add OID
-    $self->add_OID($doc_obj);
-
-    my $oid = $doc_obj->get_OID();
-    my $appletlink = "<a href=\"javascript:meldexout(\'$oid\','[TitleSafe]')\">";
-
-    $doc_obj->add_utf8_metadata ($cursection, "audiolink",  $appletlink); 
-    $doc_obj->add_utf8_metadata ($cursection, "audioicon",  "_iconaudio_"); 
-    $doc_obj->add_utf8_metadata ($cursection, "/audiolink", "</a>"); 
-
-    # process the document
-    $processor->process($doc_obj);
-}
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $filename = &util::filename_cat($base_dir, $file);
-
-    return undef unless ($filename =~ /\.((rog|mdb)(\.gz)?)$/i && (-e $filename));
-
-    my $gz = (defined $3) ? 1: 0;
-
-    print STDERR "<Processing n='$file' p='RogPlugin'>\n" if ($gli);
-    print STDERR "RogPlugin: processing $filename\n" if $processor->{'verbosity'};
-    
-    if ($gz) {
-	open (FILE, "zcat $filename |") 
-	    || die "RogPlugin::read - zcat can't open $filename\n";
-    } else {
-	open (FILE, $filename) 
-	    || die "RogPlugin::read - can't open $filename\n";
-    }
-
-    my $doc_count = 0;
-    my $dot_count = 0;
-    my $file_buffer = { line_no => 0, next_line => "", song => {} };
-
-    while ($self->read_rog_record($file_buffer))
-    {
-	$self->process_rog_record($file,$metadata,$file_buffer->{'song'},$processor);
-	$doc_count++;
-
-	last if ($maxdocs !=-1 && ($total_count+$doc_count) >= $maxdocs);
-	
-	if (($doc_count % 10) == 0)
-	{
-	    print STDERR ".";
-	    $dot_count++;
-	    print STDERR "\n" if (($dot_count % 80) == 0);
-	}
-    }
-
-    close FILE;
-
-    print STDERR "\n";
-
-    $self->{'num_processed'} = $doc_count;
-
-    return 1; # processed the file
-}
-
-1;
-
-
-
-
-
-
-
-
-
-
-
