Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/BaseMediaConverter.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/BaseMediaConverter.pm	(revision 26939)
+++ 	(revision )
@@ -1,235 +1,0 @@
-###########################################################################
-#
-# BaseMediaConverter - helper plugin that provide base functionality for 
-#                  image/video conversion using ImageMagick/ffmpeg
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package BaseMediaConverter;
-
-use PrintInfo;
-
-use convertutil;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-use gsprintf 'gsprintf';
-
-BEGIN {
-    @BaseMediaConverter::ISA = ('PrintInfo');
-}
-
-my $arguments = [
-      { 'name' => "enable_cache",
-	'desc' => "{BaseMediaConverter.enable_cache}",
-	'type' => "flag",
-	'reqd' => "no",
-	}
-
-		 ];
-
-my $options = { 'name' => "BaseMediaConverter",
-		'desc' => "{BaseMediaConverter.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args' => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists,$auxiliary) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists, $auxiliary);
-
-    return bless $self, $class;
-}
-
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    # Save base_dir for use in file cache
-    $self->{'base_dir'} = $base_dir;
-}
-
-
-sub init_cache_for_file
-{
-    my $self = shift @_;
-    my ($filename) = @_;
-
-    my $verbosity = $self->{'verbosity'};
-    my $outhandle = $self->{'outhandle'};
-    my $base_dir = $self->{'base_dir'};
-
-    my $collect_dir = $ENV{'GSDLCOLLECTDIR'};
-    $collect_dir =~ s/\\/\//g; # Work in Unix style world
-    
-##    print STDERR "**** base_dir = ", $base_dir, "\n";
- 
-    # Work out relative filename within 'base_dir'
-    $filename =~ s/\\/\//g;
-    my ($file) = ($filename =~ m/^$base_dir(.*?)$/);
-    
-    if (!defined $file) {
-	# Perhaps the filename is taken from within cache_dir area?
-	my $prev_cached_dir = $self->{'cached_dir'};
-	($file) = ($filename =~ m/^$prev_cached_dir(.*?)$/);
-    }
-
-    $file =~ s/^\/|\\//; # get rid of leading slash from relative filename
-
-
-    # Setup cached_dir and file_root
-
-    my ($file_root, $dirname, $suffix)
-	= &File::Basename::fileparse($file, "\\.[^\\.]+\$");
-
-
-    # if dirname is in collections tmp area, remove collect_dir prefix
-    $dirname =~ s/^$collect_dir//;
-
-    if ($ENV{'GSDLOS'} eq "windows") {
-	# if dirname starts with Windows drive letter, strip it off
-	$dirname =~ s/^[a-z]:\///i;
-    }
-
-    my $base_output_dir = &util::filename_cat($collect_dir,"cached",$dirname);
-
-    if (!-e $base_output_dir ) {
-	print $outhandle "Creating directory $base_output_dir\n"
-	    if ($verbosity>2);
-
-	&util::mk_all_dir($base_output_dir);
-    }
-
-    my $output_dir = &util::filename_cat($base_output_dir,$file_root);
-
-    if (!-e $output_dir) {
-	print $outhandle "Creating directory $output_dir\n"
-	    if ($verbosity>2);
-
-	&util::mk_dir($output_dir);
-    }
-
-    $self->{'cached_dir'} = $output_dir;
-    $self->{'cached_file_root'} = $file_root;    
-}
-
-
-
-sub run_general_cmd
-{
-    my $self = shift @_;
-    my ($command,$print_info) = @_;
-
-
-    if (!defined $print_info->{'verbosity'}) {
-	$print_info->{'verbosity'} = $self->{'verbosity'};
-    }
-
-    if (!defined $print_info->{'outhandle'}) {
-	$print_info->{'outhandle'} = $self->{'outhandle'};
-    }
-
-    
-    return &convertutil::run_general_cmd(@_);
-}
-
-
-sub regenerate_general_cmd
-{
-    my $self = shift @_;
-    my ($command,$ifilename,$ofilename,$print_info) = @_;
-
-    if (!defined $print_info->{'verbosity'}) {
-	$print_info->{'verbosity'} = $self->{'verbosity'};
-    }
-
-    if (!defined $print_info->{'outhandle'}) {
-	$print_info->{'outhandle'} = $self->{'outhandle'};
-    }
-
-    return &convertutil::regenerate_general_cmd(@_);
-}
-
-
-
-sub run_uncached_general_cmd
-{
-    my $self = shift @_;
-
-    my ($command,$ifilename,$ofilename,$print_info) = @_;
-
-    return $self->run_general_cmd($command,$print_info);
-}
-
-
-
-sub run_cached_general_cmd
-{
-    my $self = shift @_;
-
-    my ($command,$ifilename,$ofilename,$print_info) = @_;
-
-    if (!defined $print_info->{'verbosity'}) {
-	$print_info->{'verbosity'} = $self->{'verbosity'};
-    }
-
-    if (!defined $print_info->{'outhandle'}) {
-	$print_info->{'outhandle'} = $self->{'outhandle'};
-    }
-
-    return &convertutil::run_cached_general_cmd(@_);
-}
-
-
-
-sub autorun_general_cmd
-{
-    my $self = shift @_;
-
-    my ($command,$ifilename,$ofilename,$print_info) = @_;
-
-    my $result;
-    my $regenerated;
-    my $had_error;
-
-    if ($self->{'enable_cache'}) {
-	($regenerated,$result,$had_error)
-	    = $self->run_cached_general_cmd($command,$ifilename,$ofilename,$print_info);
-    }
-    else {
-	$regenerated = 1; # always true for a command that is always run
-	($result,$had_error)
-	    = $self->run_general_cmd($command,$print_info);
-    }
-
-    return ($regenerated,$result,$had_error);
-}
-
-
-#
-1;	
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/BasePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/BasePlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,1439 +1,0 @@
-###########################################################################
-#
-# BasePlugin.pm -- base class for all the import plugins
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999-2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package BasePlugin;
-
-use strict; 
-no strict 'subs';
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use File::Basename;
-use Encode;
-use Unicode::Normalize 'normalize';
-
-use encodings;
-use unicode;
-use doc;
-use ghtml;
-use gsprintf 'gsprintf';
-
-use PrintInfo;
-
-BEGIN {
-    @BasePlugin::ISA = ( 'PrintInfo' );
-}
-
-# the different methods that can be applied when renaming
-# imported documents and their associated files
-our $file_rename_method_list = 
-    [ { 'name' => "url",
-	'desc' => "{BasePlugin.rename_method.url}" },
-      { 'name' => "base64",
-	'desc' => "{BasePlugin.rename_method.base64}" }, 
-      { 'name' => "none",
-	'desc' => "{BasePlugin.rename_method.none}", 
-	'hiddengli' => "yes" } ];
-
-our $encoding_list =
-    [ { 'name' => "ascii",
-	'desc' => "{BasePlugin.encoding.ascii}" },
-      { 'name' => "utf8",
-	'desc' => "{BasePlugin.encoding.utf8}" },
-      { 'name' => "unicode",
-	'desc' => "{BasePlugin.encoding.unicode}" } ];
-
-
-my $e = $encodings::encodings;
-foreach my $enc (sort {$e->{$a}->{'name'} cmp $e->{$b}->{'name'}} keys (%$e)) 
-{
-    my $hashEncode =
-    {'name' => $enc,
-     'desc' => $e->{$enc}->{'name'}};
-    
-    push(@{$encoding_list},$hashEncode);
-}
-
-our $encoding_plus_auto_list = 
-    [ { 'name' => "auto",
-	'desc' => "{BasePlugin.filename_encoding.auto}" },
-      { 'name' => "auto-language-analysis",
-	'desc' => "{BasePlugin.filename_encoding.auto_language_analysis}" }, # textcat
-      { 'name' => "auto-filesystem-encoding",
-	'desc' => "{BasePlugin.filename_encoding.auto_filesystem_encoding}" }, # locale
-      { 'name' => "auto-fl",
-	'desc' => "{BasePlugin.filename_encoding.auto_fl}" }, # locale followed by textcat
-      { 'name' => "auto-lf",
-	'desc' => "{BasePlugin.filename_encoding.auto_lf}" } ]; # texcat followed by locale 
-
-push(@{$encoding_plus_auto_list},@{$encoding_list});
-
-our $oidtype_list = 
-    [ { 'name' => "auto",
-	'desc' => "{BasePlugin.OIDtype.auto}" },
-      { 'name' => "hash",
-        'desc' => "{import.OIDtype.hash}" },
-      { 'name' => "hash_on_ga_xml",
-        'desc' => "{import.OIDtype.hash_on_ga_xml}" },
-      { 'name' => "assigned",
-        'desc' => "{import.OIDtype.assigned}" },
-      { 'name' => "incremental",
-        'desc' => "{import.OIDtype.incremental}" },
-      { 'name' => "dirname",
-        'desc' => "{import.OIDtype.dirname}" } ];
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "no_blocking",
-	'desc' => "{BasePlugin.no_blocking}",
-	'type' => "flag",
-	'reqd' => "no"},
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'deft' => "",
-	'reqd' => "no" },
-      { 'name' => "store_original_file",
-	'desc' => "{BasePlugin.store_original_file}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "associate_ext",
-	'desc' => "{BasePlugin.associate_ext}",
-	'type' => "string",
-	'reqd' => "no" },
-      { 'name' => "associate_tail_re",
-	'desc' => "{BasePlugin.associate_tail_re}",
-	'type' => "string",
-	'reqd' => "no" },
-      { 'name' => "OIDtype",
-	'desc' => "{import.OIDtype}",
-	'type' => "enum",
-	'list' => $oidtype_list,
-	# leave default empty so we can tell if its been set or not - if not set will use option from import.pl
-	'deft' => "auto",
-	'reqd' => "no" },
-      { 'name' => "OIDmetadata",
-	'desc' => "{import.OIDmetadata}",
-	'type' => "metadata",
-	'deft' => "dc.Identifier",
-	'reqd' => "no" },      
-#      { 'name' => "use_as_doc_identifier",
-#	'desc' => "{BasePlugin.use_as_doc_identifier}",
-#	'type' => "string",
-#	'reqd' => "no" ,
-#	'deft' => "" } ,
-      { 'name' => "no_cover_image",
-	'desc' => "{BasePlugin.no_cover_image}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "filename_encoding",
-	'desc' => "{BasePlugin.filename_encoding}",
-	'type' => "enum",
-	'deft' => "auto",
-	'list' => $encoding_plus_auto_list,
-	'reqd' => "no" },
-      { 'name' => "smart_block",
-        'desc' => "{common.deprecated}. {BasePlugin.smart_block}",
-        'type' => "flag",
-        'reqd' => "no",
-	'hiddengli' => "yes" }, # deprecated, but leave in for old collections
-      { 'name' => "file_rename_method",
-	'desc' => "{BasePlugin.file_rename_method}",
-	'type' => "enum",
-	'deft' => &get_default_file_rename_method(), # by default rename imported files and assoc files using this encoding
-	'list' => $file_rename_method_list,
-	'reqd' => "no"
-	}
-      
-      ];
-
-
-my $options = { 'name'     => "BasePlugin",
-		'desc'     => "{BasePlugin.desc}",
-		'abstract' => "yes",
-		'inherits' => "no",
-		'args'     => $arguments };
-
-sub new {
-
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists,$auxiliary) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists,$auxiliary);
-    
-    if ($self->{'info_only'}) {
-        # don't worry about any options etc
-        return bless $self, $class;
-    }
-
-    if ($self->{'smart_block'}) {
-	print STDERR "WARNING: -smart_block option has been deprecated and is no longer useful\n";
-    }
-    $self->{'smart_block'} = undef;
-
-    my $plugin_name = (defined $pluginlist->[0]) ? $pluginlist->[0] : $class;
-    $self->{'plugin_type'} = $plugin_name;
-
-    # remove ex. from OIDmetadata iff it's the only namespace prefix
-    $self->{'OIDmetadata'} =~ s/^ex\.([^.]+)$/$1/ if defined $self->{'OIDmetadata'};
-    $self->{'num_processed'} = 0;
-    $self->{'num_not_processed'} = 0;
-    $self->{'num_blocked'} = 0;
-    $self->{'num_archives'} = 0;
-    $self->{'cover_image'} = 1; # cover image is on by default
-    $self->{'cover_image'} = 0 if ($self->{'no_cover_image'});
-    $self->{'can_process_directories'} = 0;
-    #$self->{'option_list'} = $hashArgOptLists->{"OptList"};
-    
-    my $associate_ext = $self->{'associate_ext'};
-    if ((defined $associate_ext) && ($associate_ext ne "")) {
-
-	my $associate_tail_re = $self->{'associate_tail_re'};
-	if ((defined $associate_tail_re) && ($associate_tail_re ne "")) {
-	    my $outhandle = $self->{'outhandle'};
-	    print $outhandle "Warning: can only specify 'associate_ext' or 'associate_tail_re'\n";
-	    print $outhandle "         defaulting to 'associate_tail_re'\n";
-	}
-	else {
-	    my @exts = split(/,/,$associate_ext);
-
-	    my @exts_bracketed = map { $_ = "(?:\\.$_)" } @exts;
-	    my $associate_tail_re = join("|",@exts_bracketed);
-	    $self->{'associate_tail_re'} = $associate_tail_re;
-	}
-
-	delete $self->{'associate_ext'};
-    }
-
-    return bless $self, $class;
-
-}
-
-sub merge_inheritance
-{
-    my $self = {};
-    my @child_selfs = @_;
-
-    foreach my $child_self (@child_selfs) {	
-	foreach my $key (keys %$child_self) {
-	    if (defined $self->{$key}) {
-		if ($self->{$key} ne $child_self->{$key}) {
-#		    print STDERR "Warning: Conflicting value in multiple inheritance for '$key'\n";
-#		    print STDERR "Existing stored value = $self->{$key}\n";
-#		    print STDERR "New (child) value     = $child_self->{$key}\n";
-#		    print STDERR "Keeping existing value\n";
-		    # Existing value seems to be option specified in collect.cfg
-
-		    ### $self->{$key} = $child_self->{$key};
-		    
-		}
-		else {
-##		    print STDERR "****Info: Value $self->{$key} for $key already defined through multiple inheritance as the same value\n";
-		}
-
-	    }
-	    else {
-		$self->{$key} = $child_self->{$key};
-	    }
-	}
-    }
-
-    return $self;	
-}
-
-# initialize BasePlugin options
-# if init() is overridden in a sub-class, remember to call BasePlugin::init()
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    # verbosity is passed through from the processor
-    $self->{'verbosity'} = $verbosity;
-
-    # as are the outhandle and failhandle
-    $self->{'outhandle'} = $outhandle if defined $outhandle;
-    $self->{'failhandle'} = $failhandle;
-#    $self->SUPER::init(@_);
-    
-    # set process_exp and block_exp to defaults unless they were
-    # explicitly set
-
-    if ((!$self->is_recursive()) and 
-	(!defined $self->{'process_exp'}) || ($self->{'process_exp'} eq "")) {
-
-	$self->{'process_exp'} = $self->get_default_process_exp ();
-	if ($self->{'process_exp'} eq "") {
-	    warn ref($self) . " Warning: Non-recursive plugin has no process_exp\n";
-	}
-    }
-
-    if ((!defined $self->{'block_exp'}) || ($self->{'block_exp'} eq "")) {
-	$self->{'block_exp'} = $self->get_default_block_exp ();
-    }
-
-}
-
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    if ($self->{'OIDtype'} eq "auto") {
-	# hasn't been set in the plugin, use the processor values
-	$self->{'OIDtype'} = $processor->{'OIDtype'};
-	$self->{'OIDmetadata'} = $processor->{'OIDmetadata'};
-    }
-    if ($self->{'OIDtype'} eq "hash") {
-	# should we hash on the file or on the doc xml??
-	$self->{'OIDtype'} = $self->get_oid_hash_type();
-	if ($self->{'OIDtype'} !~ /^(hash_on_file|hash_on_ga_xml)$/) {
-	    $self->{'OIDtype'} = "hash_on_file";
-	}
-    }
-}
-
-# This is called once if removeold is set with import.pl. Most plugins will do
-# nothing but if a plugin does any stuff outside of creating doc obj, then 
-# it may need to clear something.
-sub remove_all {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-}
-
-# This is called per document for docs that have been deleted from the 
-# collection. Most plugins will do nothing
-# but if a plugin does any stuff outside of creating doc obj, then it may need
-# to clear something.
-sub remove_one {
-    my $self = shift (@_);
-    
-    my ($file, $oids, $archivedir) = @_;
-    return 0 if $self->can_process_this_file($file);
-    return undef;
-}
-
-sub end {
-    # potentially called at the end of each plugin pass 
-    # import.pl only has one plugin pass, but buildcol.pl has multiple ones
-
-    my ($self) = shift (@_);
-}
-
-sub deinit {
-    # called only once, after all plugin passes have been done
-
-    my ($self) = @_;
-}
-
-# default hashing type is to hash on the original file (or converted file)
-# override this to return hash_on_ga_xml for filetypes where hashing on the 
-# file is no good eg video
-sub get_oid_hash_type {
-
-    my $self = shift (@_);
-
-    return "hash_on_file";
-}
-
-
-# this function should be overridden to return 1
-# in recursive plugins
-sub is_recursive {
-    my $self = shift (@_);
-
-    return 0; 
-}
-
-sub get_default_block_exp {
-    my $self = shift (@_);
-
-    return "";
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return "";
-}
-
-
-# rename imported files and assoc files using URL encoding by default
-# as this will work for most plugins and give more legible filenames
-sub get_default_file_rename_method() {
-    my $self = shift (@_);
-    return "url";
-}
-
-# returns this plugin's active (possibly user-selected) file_rename_method
-sub get_file_rename_method() {
-    my $self = shift (@_);
-    my $rename_method = $self->{'file_rename_method'};
-    if($rename_method) {
-	return $rename_method;
-    } else {	
-	return $self->get_default_file_rename_method();
-    }
-}
-
-# default implementation is to do nothing
-sub store_block_files {
-    
-    my $self =shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-
-}
-
-# put files to block into hash 
-sub use_block_expressions {
-
-    my $self =shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-
-    $filename_full_path = &util::upgrade_if_dos_filename($filename_full_path);
-
-    if ($self->{'block_exp'} ne "" && $filename_full_path =~ /$self->{'block_exp'}/) {
-	&util::block_filename($block_hash,$filename_full_path);
-    }
-
-}
-
-#default implementation is to block a file with same name as this, but extension jpg or JPG, if cover_images is on.
-sub block_cover_image
-{
-    my $self =shift;
-    my ($filename, $block_hash) = @_;
-
-    $filename = &util::upgrade_if_dos_filename($filename);
-
-    if ($self->{'cover_image'}) {
-	my $coverfile = $filename;
-	$coverfile =~ s/\.[^\\\/\.]+$/\.jpg/;
-
-	if (!&util::fd_exists($coverfile)) {
-	    $coverfile =~ s/jpg$/JPG/;
-	} 	
-	if (&util::fd_exists($coverfile)) {
-	    &util::block_filename($block_hash,$coverfile);
-	} 
-    }
-
-    return;
-}
-
-
-# discover all the files that should be blocked by this plugin
-# check the args ...
-sub file_block_read {
-
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
-    # Keep track of filenames with same root but different extensions
-    # Used to support -associate_ext and the more generalised
-    # -associate_tail_re
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    if (!-d $filename_full_path) {
-	$block_hash->{'all_files'}->{$file} = 1;
-    }
-
-    my $associate_tail_re = $self->{'associate_tail_re'};
-    if ((defined $associate_tail_re) && ($associate_tail_re ne "")) {
-	my ($file_prefix,$file_ext) 
-	    = &util::get_prefix_and_tail_by_regex($filename_full_path,$associate_tail_re);
-	if ((defined $file_prefix) && (defined $file_ext)) {
-	    my $shared_fileroot = $block_hash->{'shared_fileroot'};
-	    if (!defined $shared_fileroot->{$file_prefix}) {
-		my $file_prefix_rec = { 'tie_to'  => undef, 
-				        'exts'    => {} };
-		$shared_fileroot->{$file_prefix} = $file_prefix_rec;
-	    }
-	    
-	    my $file_prefix_rec = $shared_fileroot->{$file_prefix};
-
-	    if ($self->can_process_this_file($filename_full_path)) {
-		# This is the document the others should be tied to
-		$file_prefix_rec->{'tie_to'} = $file_ext;
-	    }
-	    else {
-		if ($file_ext =~ m/$associate_tail_re$/) {
-		    # this file should be associated to the main one
-		    $file_prefix_rec->{'exts'}->{$file_ext} = 1;
-		}
-	    }
-
-	}
-    }
-
-    # check block expressions
-    $self->use_block_expressions($filename_full_path, $block_hash) unless $self->{'no_blocking'};
-
-    # now check whether we are actually processing this
-    if (!-f $filename_full_path || !$self->can_process_this_file($filename_full_path)) {
-	return undef; # can't recognise
-    }
-    
-    # if we have a block_exp, then this overrides the normal 'smart' blocking
-    $self->store_block_files($filename_full_path, $block_hash) unless ($self->{'no_blocking'} || $self->{'block_exp'} ne "");
-
-    # block the cover image if there is one
-    if ($self->{'cover_image'}) {
-	$self->block_cover_image($filename_full_path, $block_hash); 
-    }
-    
-    return 1;
-}
-
-# plugins that rely on more than process_exp (eg XML plugins) can override this method
-sub can_process_this_file {
-    my $self = shift(@_);
-    my ($filename) = @_;
-
-    if (-d $filename && !$self->{'can_process_directories'}) {
-	return 0;
-    }
-
-    if ($self->{'process_exp'} ne "" && $filename =~ /$self->{'process_exp'}/) {
-	return 1;
-    }
-    return 0;
-    
-}
-
-# Even if a plugin can extract metadata in its metadata_read pass,
-# make the default return 'undef' so processing of the file continues
-# down the pipeline, so other plugins can also have the opportunity to
-# locate metadata and set it up in the extrametakeys variables that
-# are passed around.
-
-sub can_process_this_file_for_metadata {
-    my $self = shift(@_);
-
-    return undef;
-}
-
-
-# just converts path as is to utf8.
-sub filepath_to_utf8 {
-    my $self = shift (@_);  
-    my ($file, $file_encoding) = @_;
-    my $filemeta = $file;
-
-    my $filename_encoding = $self->{'filename_encoding'}; # filename encoding setting
-
-    # Whenever filename-encoding is set to any of the auto settings, we
-    # check if the filename is already in UTF8. If it is, then we're done.
-    if($filename_encoding =~ m/auto/) {
-	if(&unicode::check_is_utf8($filemeta)) 
-	{
-	    $filename_encoding = "utf8";
-	    return $filemeta;
-	} 
-    }
-    
-    # Auto setting, but filename is not utf8
-    if ($filename_encoding eq "auto") 
-    {
-	# try textcat
-	$filename_encoding = $self->textcat_encoding($filemeta);
-	
-	# check the locale next
-	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
-	
-	
-	# now try the encoding of the document, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-
-    }
-
-    elsif ($filename_encoding eq "auto-language-analysis") 
-    {	
-	$filename_encoding = $self->textcat_encoding($filemeta);
-
-	# now try the encoding of the document, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	} 
-    }
-
-    elsif ($filename_encoding eq "auto-filesystem-encoding") 
-    {	
-	# try locale
-	$filename_encoding = $self->locale_encoding();
-    }
-
-    elsif ($filename_encoding eq "auto-fl") 
-    {
-	# filesystem-encoding (locale) then language-analysis (textcat)
-	$filename_encoding = $self->locale_encoding();
-	
-	# try textcat
-	$filename_encoding = $self->textcat_encoding($filemeta) if $filename_encoding eq "undefined";
-	
-	# else assume filename encoding is encoding of file content, if that's available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-    }
-    
-    elsif ($filename_encoding eq "auto-lf") 
-    {
-	# language-analysis (textcat) then filesystem-encoding (locale)
-	$filename_encoding = $self->textcat_encoding($filemeta);
-	
-	# guess filename encoding from encoding of file content, if available
-	if ($filename_encoding eq "undefined" && defined $file_encoding) {
-	    $filename_encoding = $file_encoding;
-	}
-
-	# try locale
-	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
-    }
-    
-    # if still undefined, use utf8 as fallback
-    if ($filename_encoding eq "undefined") {
-	$filename_encoding = "utf8";
-    }
-
-    #print STDERR "**** UTF8 encoding the filename $filemeta ";
-    
-    # if the filename encoding is set to utf8 but it isn't utf8 already--such as when
-    # 1. the utf8 fallback is used, or 2. if the system locale is used and happens to
-    # be always utf8 (in which case the filename's encoding is also set as utf8 even 
-    # though the filename need not be if it originates from another system)--in such
-    # cases attempt to make the filename utf8 to match.
-    if($filename_encoding eq "utf8" && !&unicode::check_is_utf8($filemeta)) {
-	&unicode::ensure_utf8(\$filemeta);
-    }
-
-    # convert non-unicode encodings to utf8
-    if ($filename_encoding !~ m/(?:ascii|utf8|unicode)/) {
-	$filemeta = &unicode::unicode2utf8(
-					   &unicode::convert2unicode($filename_encoding, \$filemeta)
-					   );
-    }
-
-    #print STDERR " from encoding $filename_encoding -> $filemeta\n";
-    return $filemeta;
-}
-
-# gets the filename with no path, converts to utf8, and then dm safes it.
-# filename_encoding set by user
-sub filename_to_utf8_metadata
-{
-    my $self = shift (@_);  
-    my ($file, $file_encoding) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    print $outhandle "****!!!!**** BasePlugin::filename_to_utf8_metadata now deprecated\n";
-    my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(0);
-    print $outhandle "Calling method: $cfilename:$cline $cpackage->$csubr\n";
-
-
-    my ($filemeta) = $file =~ /([^\\\/]+)$/; # getting the tail of the filepath (skips all string parts containing slashes upto the end)
-    $filemeta = $self->filepath_to_utf8($filemeta, $file_encoding);
-
-    return $filemeta;
-}
-
-sub locale_encoding {
-    my $self = shift(@_);
-    
-    if (!defined $self->{'filesystem_encoding'}) {
-	$self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
-    }
-
-    #print STDERR "*** filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
-    return $self->{'filesystem_encoding'}; # can be the string "undefined"
-}
-
-sub textcat_encoding {
-    my $self = shift(@_);
-    my ($filemeta) = @_;
-
-    # analyse filenames without extensions and digits (and trimmed of
-    # surrounding whitespace), so that irrelevant chars don't confuse
-    # textcat
-    my $strictfilemeta = $filemeta;
-    $strictfilemeta =~ s/\.[^\.]+$//g;
-    $strictfilemeta =~ s/\d//g;
-    $strictfilemeta =~ s/^\s*//g;
-    $strictfilemeta =~ s/\s*$//g;
-    
-    my $filename_encoding = $self->encoding_from_language_analysis($strictfilemeta);
-    if(!defined $filename_encoding) {
-	$filename_encoding = "undefined";
-    }
-
-    return $filename_encoding; # can be the string "undefined"
-}
-
-# performs textcat
-sub encoding_from_language_analysis {
-    my $self = shift(@_);
-    my ($text) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $best_encoding = undef;
-    
-    # get the language/encoding of the textstring using textcat
-    require textcat;  # Only load the textcat module if it is required
-    $self->{'textcat'} = new textcat() unless defined($self->{'textcat'});
-    my $results = $self->{'textcat'}->classify_cached_filename(\$text);
-
-
-    if (scalar @$results < 0) { 
-	return undef;
-    }
-    
-    # We have some results, we choose the first
-    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
-    
-    $best_encoding = $encoding;
-    if (!defined $best_encoding) {
-	return undef;
-    } 
-    
-    if (defined $best_encoding && $best_encoding =~ m/^iso_8859/ && &unicode::check_is_utf8($text)) {
-	# the text is valid utf8, so assume that's the real encoding (since textcat is based on probabilities)
-	$best_encoding = 'utf8';
-    }
-    
-    
-    # check for equivalents where textcat doesn't have some encodings...
-    # eg MS versions of standard encodings
-    if (defined $best_encoding && $best_encoding =~ /^iso_8859_(\d+)/) {
-	my $iso = $1; # which variant of the iso standard?
-	# iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
-	if ($text =~ /[\x80-\x9f]/) {
-	    # Western Europe
-	    if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
-	    elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
-	    elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
-	    elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
-	    elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
-	    elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
-	    elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
-	}
-    }
-    
-    if (defined $best_encoding && $best_encoding !~ /^(ascii|utf8|unicode)$/ &&
-	!defined $encodings::encodings->{$best_encoding}) 
-    {
-	if ($self->{'verbosity'}) { 
-	    gsprintf($outhandle, "BasePlugin: {ReadTextFile.unsupported_encoding}\n", $text, $best_encoding, "undef");
-	}
-	$best_encoding = undef;
-    }
-    
-    return $best_encoding;
-}
-
-# uses locale
-sub get_filesystem_encoding 
-{
-
-    my $self = shift(@_);
-
-    my $outhandle = $self->{'outhandle'};
-    my $filesystem_encoding = undef;
-
-    eval {
-	# Works for Windows as well, returning the DOS code page in use	
-	use POSIX qw(locale_h);
-	
-	# With only one parameter, setlocale retrieves the 
-	# current value
-	my $current_locale = setlocale(LC_CTYPE);
-	
-	my $char_encoding = undef;
-	if ($current_locale =~ m/\./) {
-	    ($char_encoding) = ($current_locale =~ m/^.*\.(.*?)$/);
-	    $char_encoding = lc($char_encoding);
-	}
-	else {
-	    if ($current_locale =~ m/^(posix|c)$/i) {
-		$char_encoding = "ascii";
-	    }
-	}
-
-	if (defined $char_encoding) {
-	    if ($char_encoding =~ m/^(iso)(8859)-?(\d{1,2})$/) {
-		$char_encoding = "$1\_$2\_$3";
-	    }
-
-	    $char_encoding =~ s/-/_/g;
-	    $char_encoding =~ s/^utf_8$/utf8/;
-	    
-	    if ($char_encoding =~ m/^\d+$/) {
-		if (defined $encodings::encodings->{"windows_$char_encoding"}) {
-		    $char_encoding = "windows_$char_encoding";
-		}
-		elsif (defined $encodings::encodings->{"dos_$char_encoding"}) {
-		    $char_encoding = "dos_$char_encoding";
-		}
-	    }
-	    
-	    if (($char_encoding =~ m/(?:ascii|utf8|unicode)/) 
-		|| (defined $encodings::encodings->{$char_encoding})) {
-		$filesystem_encoding = $char_encoding;
-	    }
-	    else {
-		print $outhandle "Warning: Unsupported character encoding '$char_encoding' from locale '$current_locale'\n";
-	    }
-	}
-	
-
-    };
-    if ($@) {
-	print $outhandle "$@\n";
-	print $outhandle "Warning: Unable to establish locale.  Will assume filesystem is UTF-8\n";
-	
-    }
-
-    return $filesystem_encoding;
-}
-
-
-
-sub deduce_filename_encoding
-{
-    my $self = shift (@_);  
-    my ($file,$metadata,$plugin_filename_encoding) = @_;
-
-    my $gs_filename_encoding = $metadata->{"gs.filenameEncoding"};
-    my $deduced_filename_encoding = undef;
-    
-    # Start by looking for manually assigned metadata
-    if (defined $gs_filename_encoding) {
-	if (ref ($gs_filename_encoding) eq "ARRAY") {
-	    my $outhandle = $self->{'outhandle'};
-	    
-	    $deduced_filename_encoding = $gs_filename_encoding->[0];
-	    
-	    my $num_vals = scalar(@$gs_filename_encoding);
-	    if ($num_vals>1) {
-		print $outhandle "Warning: gs.filenameEncoding multiply defined for $file\n";
-		print $outhandle "         Selecting first value: $deduced_filename_encoding\n";
-	    }
-	} 
-	else {
-	    $deduced_filename_encoding = $gs_filename_encoding;
-	}
-    }
-    
-    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-	# Look to see if plugin specifies this value
-
-	if (defined $plugin_filename_encoding) {
-	    # First look to see if we're using any of the "older" (i.e. deprecated auto-... plugin options)
-	    if ($plugin_filename_encoding =~ m/^auto-.*$/) {
-		my $outhandle = $self->{'outhandle'};
-		print $outhandle "Warning: $plugin_filename_encoding is no longer supported\n";
-		print $outhandle "         default to 'auto'\n";
-		$self->{'filename_encoding'} = $plugin_filename_encoding = "auto";
-	    }
-	    
-	    if ($plugin_filename_encoding ne "auto") {
-		# We've been given a specific filenamne encoding
-		# => so use it!
-		$deduced_filename_encoding = $plugin_filename_encoding;
-	    }
-	}
-    }
-    
-    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-
-	# Look to file system to provide a character encoding
-
-	# If Windows NTFS, then -- assuming we work with long file names got through
-	# Win32::GetLongFilePath() -- then the underlying file system is UTF16
-
-	if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
-	    # Can do better than working with the DOS character encoding returned by locale	    
-	    $deduced_filename_encoding = "unicode";
-	}
-	else {
-	    # Unix of some form or other
-
-	    # See if we can determine the file system encoding through locale
-	    $deduced_filename_encoding = $self->locale_encoding();
-	    
-	    # if locale shows us filesystem is utf8, check to see filename is consistent
-	    # => if not, then we have an "alien" filename on our hands
-
-	    if (defined $deduced_filename_encoding && $deduced_filename_encoding =~ m/^utf-?8$/i) {
-		if (!&unicode::check_is_utf8($file)) {
-		    # "alien" filename, so revert
-		    $deduced_filename_encoding = undef;
-		}
-	    }
-	}
-    }
-    
-#    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
-#		# Last chance, apply textcat to deduce filename encoding
-#		$deduced_filename_encoding = $self->textcat_encoding($file);
-#    }
-
-    if ($self->{'verbosity'}>3) {
-	my $outhandle = $self->{'outhandle'};
-
-	if (defined $deduced_filename_encoding) {
-	    print $outhandle "  Deduced filename encoding as: $deduced_filename_encoding\n";
-	}
-	else {
-	    print $outhandle "  No filename encoding deduced\n";
-	}
-    }
-    
-    return $deduced_filename_encoding;
-}
-
-
-
-
-# Notionally written to be called once for each document, it is however safe to
-# call multiple times (as in the case of ImagePlugin) which calls this later on
-# after the original image has potentially been converted to a *new* source image
-# format (e.g. TIFF to PNG)
-
-sub set_Source_metadata {
-    my $self = shift (@_);  
-    my ($doc_obj, $raw_filename, $filename_encoding, $section) = @_;
-    
-    # 1. Sets the filename (Source) for display encoded as Unicode if possible,
-    #    and (as a fallback) using %xx if not for non-ascii chars
-    # 2. Sets the url ref (SourceFile) to the URL encoded version
-    #    of filename for generated files
-    
-    my ($unused_full_rf, $raw_file) = &util::get_full_filenames("", $raw_filename);
-
-    my $this_section = (defined $section)? $section : $doc_obj->get_top_section();
-
-    my $octet_file = $raw_file;
-
-    # UTF-8 version of filename
-#    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-#	print STDERR "**** Setting Source Metadata given: $octet_file\n";
-#    }
-    
-    # Deal with (on Windows) raw filenames that are in their
-    # abbreviated DOS form
-
-    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
-	if ((defined $filename_encoding) && ($filename_encoding eq "unicode")) {
-	    if (-e $raw_filename) {
-		my $unicode_filename = Win32::GetLongPathName($raw_filename);
-		
-		my $unused_full_uf;
-		($unused_full_uf, $octet_file) = &util::get_full_filenames("", $unicode_filename);
-	    }
-	}
-    }
-
-    my $url_encoded_filename;
-    if ((defined $filename_encoding) && ($filename_encoding ne "ascii")) {
-	# => Generate a pretty print version of filename that is mapped to Unicode
-	
-	# Use filename_encoding to map raw filename to a Perl unicode-aware string 
-	$url_encoded_filename = decode($filename_encoding,$octet_file);		
-    }
-    else {
-	# otherwise generate %xx encoded version of filename for char > 127
-	$url_encoded_filename = &unicode::raw_filename_to_url_encoded($octet_file);
-    }
-    
-#    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-#	print STDERR "****** saving Source as:             $url_encoded_filename\n";
-#    }
-
-    # In the case of converted files and (generalized) exploded documents, there
-    # will already be a source filename => store as OrigSource before overriding
-    my $orig_source = $doc_obj->get_metadata_element ($this_section, "Source");
-    if ((defined $orig_source) && ($orig_source !~ m/^\s*$/)) {
-	$doc_obj->set_utf8_metadata_element($this_section, "OrigSource", $orig_source); 
-    }
-        
-    # Source is the UTF8 display name - not necessarily the name of the file on the system
-    if ($ENV{'GSDLOS'} =~ m/^darwin$/i) {
-	# on Darwin want all display strings to be in composed form, then can search on that
-	$url_encoded_filename = normalize('C', $url_encoded_filename); # Normalisation Form 'C' (composition)
-    }
-    $doc_obj->set_utf8_metadata_element($this_section, "Source", $url_encoded_filename); 
-
-    
-    my $renamed_raw_file = &util::rename_file($raw_file, $self->{'file_rename_method'});
-    # If using URL encoding, then SourceFile is the url-reference to url-encoded
-    # renamed_raw_url: it's a url that refers to the actual file on the system
-    my $renamed_raw_url = &unicode::filename_to_url($renamed_raw_file);
-
-    $doc_obj->set_utf8_metadata_element($this_section, "SourceFile",
-					$renamed_raw_url);
-
-#    if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
-#	print STDERR "****** saving SourceFile as:         $renamed_raw_url\n";
-#    }
-}
-
-# this should be called by all plugins to set the oid of the doc obj, rather
-# than calling doc_obj->set_OID directly
-sub add_OID {
-    my $self = shift (@_);  
-    my ($doc_obj) = @_;
-
-    $doc_obj->set_OIDtype($self->{'OIDtype'}, $self->{'OIDmetadata'});
-
-    # see if there is a plugin specific set_OID function
-    if (defined ($self->can('set_OID'))) {
-	$self->set_OID(@_); # pass through doc_obj and any extra arguments
-    }
-    else {
-	# use the default set_OID() in doc.pm
-	$doc_obj->set_OID();
-    }
-
-}
-
-# The BasePlugin read_into_doc_obj() function. This function does all the
-# right things to make general options work for a given plugin.  It doesn't do anything with the file other than setting reads in
-# a file and sets up a slew of metadata all saved in doc_obj, which
-# it then returns as part of a tuple (process_status,doc_obj)
-#
-# Much of this functionality used to reside in read, but it was broken
-# down into a supporting routine to make the code more flexible.  
-#
-# recursive plugins (e.g. RecPlug) and specialized plugins like those
-# capable of processing many documents within a single file (e.g.
-# GMLPlug) will normally want to implement their own version of
-# read_into_doc_obj()
-#
-# Note that $base_dir might be "" and that $file might 
-# include directories
-
-# currently blocking has been done before it gets here - does this affect secondary plugin stuff??
-sub read_into_doc_obj {
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    # should we move this to read? What about secondary plugins?
-    my $pp_file = &util::prettyprint_file($base_dir,$file,$gli);
-    print STDERR "<Processing n='$file' p='$self->{'plugin_type'}'>\n" if ($gli);
-    print $outhandle "$self->{'plugin_type'} processing $pp_file\n"
-	if $self->{'verbosity'} > 1;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    
-    # create a new document
-    my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-    my $top_section = $doc_obj->get_top_section();
-
-    $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
-    
-
-    my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj,$filename_full_path,$filename_encoding,$top_section);
-
-    # plugin specific stuff - what args do we need here??
-    unless (defined ($self->process($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli))) {
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1;
-    }
-    
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    my $section = $doc_obj->get_top_section();
-    # can we merge these two methods??
-    $self->add_associated_files($doc_obj, $filename_full_path);
-    $self->extra_metadata ($doc_obj, $section, $metadata);
-    $self->auto_extract_metadata($doc_obj);
-
-    # if we haven't found any Title so far, assign one
-    # this was shifted to here from inside read()
-    $self->title_fallback($doc_obj,$section,$filename_no_path);
-    
-    $self->add_OID($doc_obj);
-    
-    $self->post_process_doc_obj($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli);
-    return (1,$doc_obj);
-}
-
-sub post_process_doc_obj {
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    return 1;
-}
-
-sub add_dummy_text {
-    my $self = shift(@_);
-    my ($doc_obj, $section) = @_;
-
-    # add NoText metadata so we can hide this dummy text in format statements
-    $doc_obj->add_metadata($section, "NoText", "1");
-    $doc_obj->add_text($section, &gsprintf::lookup_string("{BasePlugin.dummy_text}",1));
-    
-}
-
-# does nothing. Can be overridden by subclass
-sub auto_extract_metadata {
-    my $self = shift(@_);
-    my ($doc_obj) = @_;
-}
-
-# adds cover image, associate_file options stuff. Should be called by sub class
-# read_into_doc_obj
-sub add_associated_files {
-    my $self = shift(@_);
-    # whatis filename??
-    my ($doc_obj, $filename) = @_;
-    
-    # add in the cover image
-    if ($self->{'cover_image'}) {
-	$self->associate_cover_image($doc_obj, $filename);
-    }
-    # store the original (used for eg TextPlugin to store the original for OAI)
-    if ($self->{'store_original_file'}) {
-	$self->associate_source_file($doc_obj, $filename);
-    }
-    
-
-}
-
-# implement this if you are extracting metadata for other documents
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-    
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file_for_metadata($filename_full_path);
-
-    return 1; # we recognise the file, but don't actually do anything with it
-}
-
-
-# The BasePlugin read() function. This function calls read_into_doc_obj()
-# to ensure all the right things to make general options work for a
-# given plugin are done. It then calls the process() function which
-# does all the work specific to a plugin (like the old read functions
-# used to do). Most plugins should define their own process() function
-# and let this read() function keep control.  
-#
-# recursive plugins (e.g. RecPlug) and specialized plugins like those
-# capable of processing many documents within a single file (e.g.
-# GMLPlug) might want to implement their own version of read(), but
-# more likely need to implement their own version of read_into_doc_obj()
-#
-# Return number of files processed, undef if can't recognise, -1 if can't
-# process
-
-sub read {
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    return undef unless $self->can_process_this_file($filename_full_path);
-    
-    my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
-    
-    if ((defined $process_status) && ($process_status == 1)) {
-	
-	# process the document
-	$processor->process($doc_obj);
-
-	$self->{'num_processed'} ++;
-	undef $doc_obj; 
-    }
-    # delete any temp files that we may have created
-    $self->clean_up_after_doc_obj_processing();
-
-
-    # if process_status == 1, then the file has been processed.
-    return $process_status;
-
-}
-
-# returns undef if file is rejected by the plugin
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    gsprintf(STDERR, "BasePlugin::process {common.must_be_implemented}\n");
-
-    my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
-    print STDERR "Calling method: $cfilename:$cline $cpackage->$csubr\n";
-
-    die "\n";
-
-    return undef; # never gets here
-}
-
-# overwrite this method to delete any temp files that we have created
-sub clean_up_after_doc_obj_processing {
-    my $self = shift(@_);
-
-}
-
-# write_file -- used by ConvertToPlug, for example in post processing
-#
-# where should this go, is here the best place??
-sub utf8_write_file {
-    my $self = shift (@_);
-    my ($textref, $filename) = @_;
-    
-    if (!open (FILE, ">:utf8", $filename)) {
-	gsprintf(STDERR, "ConvertToPlug::write_file {ConvertToPlug.could_not_open_for_writing} ($!)\n", $filename);
-	die "\n";
-    }
-    print FILE $$textref;
-    
-    close FILE;
-}
-
-
-sub filename_based_title
-{
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    my $file_derived_title = $file;
-    $file_derived_title =~ s/_/ /g;
-    $file_derived_title =~ s/\.[^.]+$//;
-
-    return $file_derived_title;
-}
-
-
-sub title_fallback
-{
-    my $self = shift (@_);
-    my ($doc_obj,$section,$file) = @_;
-
-    if (!defined $doc_obj->get_metadata_element ($section, "Title") 
-	|| $doc_obj->get_metadata_element($section, "Title") eq "") {
-
-	my $source_file = $doc_obj->get_metadata_element($section, "Source");
-	my $file_derived_title;
-	if (defined $source_file) {
-	    $file_derived_title =  $self->filename_based_title($source_file);
-	}
-	else {
-	    # pp = pretty print
-	    my $pp_file = (defined $source_file) ? $source_file : $file;
-
-	    my $raw_title = $self->filename_based_title($file);
-	    my $file_derived_title = &unicode::raw_filename_to_url_encoded($raw_title);
-	}
-
-
-	if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
-	    $doc_obj->add_utf8_metadata ($section, "Title", $file_derived_title);
-	}
-	else {
-	    $doc_obj->set_utf8_metadata_element ($section, "Title", $file_derived_title);
-	} 
-    }
-    
-}
-
-# add any extra metadata that's been passed around from one
-# plugin to another.
-# extra_metadata uses add_utf8_metadata so it expects metadata values
-# to already be in utf8
-sub extra_metadata {
-    my $self = shift (@_);
-    my ($doc_obj, $cursection, $metadata) = @_;
-
-    my $associate_tail_re = $self->{'associate_tail_re'};
-
-    foreach my $field (keys(%$metadata)) {
-	# $metadata->{$field} may be an array reference
-	if ($field eq "gsdlassocfile_tobe") {
-	    # 'gsdlassocfile_tobe' is artificially introduced metadata
-	    # that is used to signal that certain additional files should
-	    # be tied to this document.  Useful in situations where a
-	    # metadata pass in the plugin pipeline works out some files
-	    # need to be associated with a document, but the document hasn't
-	    # been formed yet.
-	    my $equiv_form = "";
-	    foreach my $gaf (@{$metadata->{$field}}) {
-		my ($full_filename,$mimetype) = ($gaf =~ m/^(.*):(.*):$/);
-		my ($tail_filename) = ($full_filename =~ /^.*[\/\\](.+?)$/);
-		
-		# we need to make sure the filename is valid utf-8 - we do 
-		# this by url or base64 encoding it
-		# $tail_filename is the name that we store the file as
-		$tail_filename = &util::rename_file($tail_filename, $self->{'file_rename_method'});
-		$doc_obj->associate_file($full_filename,$tail_filename,$mimetype);
-		$doc_obj->associate_source_file($full_filename);
-		# If the filename is url_encoded, we need to encode the % signs 
-		# in the filename, so that it works in a url
-		my $url_tail_filename = &unicode::filename_to_url($tail_filename);
-		# work out extended tail extension (i.e. matching tail re)
-
-		my ($file_prefix,$file_extended_ext) 
-		    = &util::get_prefix_and_tail_by_regex($tail_filename,$associate_tail_re);
-		my ($pre_doc_ext) = ($file_extended_ext =~ m/^(.*)\..*$/);
-		my ($doc_ext) = ($tail_filename =~ m/^.*\.(.*)$/);
-
-##		my $start_doclink = "<a href=\"_httpprefix_/collect/[collection]/index/assoc/{Or}{[parent(Top):assocfilepath],[assocfilepath]}/$url_tail_filename\">";
-		my $start_doclink = "<a href=\'_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/$url_tail_filename\'>";
-		my $start_doclink_gs3 = "<a href=\'_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/$url_tail_filename\'>";
-
-		my $srcicon = "_icon".$doc_ext."_";
-		my $end_doclink = "</a>";
-		
-		my $assoc_form = "$start_doclink\{If\}{$srcicon,$srcicon,$doc_ext\}$end_doclink";
-
-		$doc_obj->add_utf8_metadata ($cursection, "equivDocLink", $start_doclink);
-		$doc_obj->add_utf8_metadata ($cursection, "equivDocIcon", $srcicon);
-		$doc_obj->add_utf8_metadata ($cursection, "/equivDocLink", $end_doclink);
-
-		if (defined $pre_doc_ext && $pre_doc_ext ne "") {
-		    # for metadata such as [mp3._edited] [mp3._full] ...
-		    $doc_obj->add_utf8_metadata ($cursection, "$doc_ext.$pre_doc_ext", $assoc_form); 
-		}
-
-		# for multiple metadata such as [mp3.assoclink]
-		$doc_obj->add_utf8_metadata ($cursection, "$doc_ext.assoclink", $assoc_form); 
-
-		$equiv_form .= " $assoc_form";	
-	    }
-	    $doc_obj->add_utf8_metadata ($cursection, "equivlink", $equiv_form); 
-	}
-	elsif ($field eq "gsdlzipfilename") {
-	    # special case for when files have come out of a zip. source_path 
-	    # (used for archives dbs and keeping track for incremental import)
-	    # must be set to the zip file name
-	    my $zip_filename = $metadata->{$field};
-	    # overwrite the source_path
-	    $doc_obj->set_source_path($zip_filename);
-	    # and set the metadata
-	    $zip_filename = &util::filename_within_collection($zip_filename);
-	    $zip_filename = $doc_obj->encode_filename($zip_filename, $self->{'file_rename_method'});
-	    $doc_obj->add_utf8_metadata ($cursection, $field, $zip_filename);
-	}
-	elsif (ref ($metadata->{$field}) eq "ARRAY") {
-	    map { 
-		$doc_obj->add_utf8_metadata ($cursection, $field, $_); 
-	    } @{$metadata->{$field}};
-	} else {
-	    $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
-	}
-    }
-}
-
-
-sub compile_stats {
-    my $self = shift(@_);
-    my ($stats) = @_;
-
-    $stats->{'num_processed'} += $self->{'num_processed'};
-    $stats->{'num_not_processed'} += $self->{'num_not_processed'};
-    $stats->{'num_archives'} += $self->{'num_archives'};
-
-}
-sub associate_source_file {
-    my $self = shift(@_);
-    
-    my ($doc_obj, $filename) = @_;
-    my $cursection = $doc_obj->get_top_section();
-    my $assocfilename = $doc_obj->get_assocfile_from_sourcefile();
-    
-    $doc_obj->associate_file($filename, $assocfilename, undef, $cursection);
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_utf8_metadata ($cursection, "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_utf8_metadata ($cursection, "srclinkFile", $doc_obj->get_sourcefile());
-}
-
-sub associate_cover_image {
-    my $self = shift(@_);
-    my ($doc_obj, $filename) = @_;
-
-    my $upgraded_filename = &util::upgrade_if_dos_filename($filename);
-
-    $filename =~ s/\.[^\\\/\.]+$/\.jpg/;
-    $upgraded_filename =~ s/\.[^\\\/\.]+$/\.jpg/;
-
-    if (exists $self->{'covers_missing_cache'}->{$upgraded_filename}) {
-	# don't stat() for existence e.g. for multiple document input files
-	# (eg SplitPlug)
-	return;
-    }
-
-    my $top_section=$doc_obj->get_top_section();
-
-    if (&util::fd_exists($upgraded_filename)) {
-	$doc_obj->associate_source_file($filename);
-    	$doc_obj->associate_file($filename, "cover.jpg", "image/jpeg");
-	$doc_obj->add_utf8_metadata($top_section, "hascover",  1);
-    } else {
-	my $upper_filename = $filename;
-	my $upgraded_upper_filename = $upgraded_filename;
-
-	$upper_filename =~ s/jpg$/JPG/;
-	$upgraded_upper_filename =~ s/jpg$/JPG/;
-
-	if (&util::fd_exists($upgraded_upper_filename)) {
-	    $doc_obj->associate_source_file($upper_filename);
-	    $doc_obj->associate_file($upper_filename, "cover.jpg",
-				     "image/jpeg");
-	    $doc_obj->add_utf8_metadata($top_section, "hascover",  1);
-	} else {
-	    # file doesn't exist, so record the fact that it's missing so
-	    # we don't stat() again (stat is slow)
-	    $self->{'covers_missing_cache'}->{$upgraded_filename} = 1;
-	}
-    }
-
-}
-
-
-# Overridden by exploding plugins (eg. ISISPlug)
-sub clean_up_after_exploding
-{
-    my $self = shift(@_);
-}
-
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/DateExtractor.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DateExtractor.pm	(revision 26939)
+++ 	(revision )
@@ -1,104 +1,0 @@
-###########################################################################
-#
-# DateExtractor - helper plugin that extracts historical dates from text
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package DateExtractor;
-
-use DateExtract;
-use PrintInfo;
-use strict;
-
-BEGIN {
-    @DateExtractor::ISA = ('PrintInfo');
-}
-
-my $arguments = [
-      { 'name' => "extract_historical_years",
-	'desc' => "{DateExtractor.extract_historical_years}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "maximum_year",
-	'desc' => "{DateExtractor.maximum_year}",
-	'type' => "int",
-	'deft' => (localtime)[5]+1900,
-	'char_length' => "4",
-	#'range' => "2,100",
-	'reqd' => "no"},
-      { 'name' => "maximum_century",
-	'desc' => "{DateExtractor.maximum_century}",
-	'type' => "string",
-	'deft' => "-1",
-	'reqd' => "no" },
-      { 'name' => "no_bibliography",
-	'desc' => "{DateExtractor.no_bibliography}",
-	'type' => "flag",
-	'reqd' => "no"},
-		 ];
-
-my $options = { 'name'     => "DateExtractor",
-		'desc'     => "{DateExtractor.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists, 1);
-
-    return bless $self, $class;
-
-}
-
-
-# extract metadata
-sub extract_date_metadata {
-
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-    
-    if($self->{'extract_historical_years'}) {
-	my $thissection = $doc_obj->get_top_section();
-	while (defined $thissection) {
-
-	    my $text = $doc_obj->get_text($thissection);
-	    &DateExtract::get_date_metadata($text, $doc_obj, 
-					    $thissection, 
-					    $self->{'no_bibliography'}, 
-					    $self->{'maximum_year'}, 
-					    $self->{'maximum_century'});
-	    $thissection = $doc_obj->get_next_section ($thissection);
-	}
-    }
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ExcelPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ExcelPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,192 +1,0 @@
-###########################################################################
-#
-# ExcelPlugin.pm -- plugin for importing Microsoft Excel files.
-#  (basic version supports versions 95 and 97)
-#  (through OpenOffice extension, supports all contempoary formats)
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2002 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ExcelPlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-use gsprintf 'gsprintf';
-
-use AutoLoadConverters;
-use ConvertBinaryFile;
-
-sub BEGIN {
-    @ExcelPlugin::ISA = ('ConvertBinaryFile', 'AutoLoadConverters');
-}
-
-my $openoffice_available = 0;
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => "&get_default_process_exp()"  # delayed (see below)
-	}
-      ];
-
-my $options = { 'name'     => "ExcelPlugin",
-		'desc'     => "{ExcelPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in Excel format can be replaced with GS-generated html
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-    
-    # this bit needs to happen later after the arguments array has been 
-    # finished - used for parsing the input args.
-    # push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    # this one needs to go in first, to get the print info in the right order
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $auto_converter_self = new AutoLoadConverters($pluginlist,$inputargs,$hashArgOptLists,["OpenOfficeConverter"],1);
-
-    if ($auto_converter_self->{'openoffice_available'}) {
-	$openoffice_available = 1;
-    } 
-
-    # evaluate the default for process_exp  - it needs to be delayed till here so we know if openoffice is available or not. But needs to be done before parsing the args.
-    foreach my $a (@$arguments) {
-	if ($a->{'name'} eq "process_exp") {
-	    my $eval_expr = $a->{'deft'};
-	    $a->{'deft'} = eval "$eval_expr";
-	    last;
-	}
-    }
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    my $cbf_self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-    my $self = BasePlugin::merge_inheritance($auto_converter_self, $cbf_self);
-
-    
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    $self = bless $self, $class;
-    $self->{'file_type'} = "Excel";
-
-    my $outhandle = $self->{'outhandle'};
-
-    # check convert_to
-    if ($self->{'convert_to'} eq "auto") {
-	$self->{'convert_to'} = "html";
-    }
-
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
-
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    push(@$specific_options,"-extract_language") if $self->{'extract_language'};
-    push(@$specific_options, "-file_rename_method", "none");
-
-    if ($secondary_plugin_name eq "HTMLPlugin") {
-	push(@$specific_options, "-processing_tmp_files");
-    }
-    
-    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
-    return $self;    
-}
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    if ($openoffice_available) {
-	return q^(?i)\.(xls|xlsx|ods)$^;
-    }
-
-    return q^(?i)\.xls$^;
-}
- 
-sub init {
-    my $self = shift (@_);
-
-    # ConvertBinaryFile init
-    $self->SUPER::init(@_);
-    $self->AutoLoadConverters::init(@_);
-
-}
-
-sub begin {
-    my $self = shift (@_);
-
-    $self->AutoLoadConverters::begin(@_);
-    $self->SUPER::begin(@_);
-
-}
-
-sub deinit {
-    my $self = shift (@_);
-    
-    $self->AutoLoadConverters::deinit(@_);
-    $self->SUPER::deinit(@_);
-
-}
-
-sub tmp_area_convert_file {
-
-    my $self = shift (@_);
-    return $self->AutoLoadConverters::tmp_area_convert_file(@_);
-
-}
-   
-sub convert_post_process_old
-{
-    my $self = shift (@_);
-    my ($conv_filename) = @_;
-    
-    my $outhandle=$self->{'outhandle'};
-    
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
-    
-    # read in file ($text will be in utf8)
-    my $text = "";
-    $self->read_file ($conv_filename, $encoding, $language, \$text);
-    
-    # turn any high bytes that aren't valid utf-8 into utf-8.
-    #unicode::ensure_utf8(\$text);
-    
-    # Write it out again!
-    #$self->utf8_write_file (\$text, $conv_filename);
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/FOXPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/FOXPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,303 +1,0 @@
-###########################################################################
-#
-# FOXPlugin.pm
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# plugin to process a Foxbase dbt file. This plugin provides the basic
-# functionality to read in the dbt and dbf files and process each record.
-# This general plugin should be overridden for a particular database to process
-# the appropriate fields in the file.
-
-package FOXPlugin;
-
-use BasePlugin;
-use util;
-use doc;
-use unicode;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-
-sub BEGIN {
-    @FOXPlugin::ISA = ('BasePlugin');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() },
-      { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_block_exp() } ];
-
-my $options = { 'name'     => "FOXPlugin",
-		'desc'     => "{FOXPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.dbf$^;
-}
-
-#dbt files are processed at the same time as dbf files
-sub get_default_block_exp {
-    my $self = shift (@_);
-
-    return q^(?i)\.dbt$^;
-}
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
- 
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-
-    print STDERR "<Processing n='$file' p='FOXPlugin'>\n" if ($gli);
-    print STDERR "FOXPlugin: processing $file\n" if $self->{'verbosity'} > 1;
-
-    my ($parent_dir) = $filename_full_path =~ /^(.*)\/[^\/]+\.dbf$/i;
-
-    # open the file
-    if (!open (FOXBASEIN, $filename_full_path)) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='Could not read $filename_full_path'>\n";
-	}
-	print STDERR "FOXPlugin::read - couldn't read $filename_full_path\n";
-	return -1; # error in processing
-    }
-
-    # read in the database header
-    my ($temp, %dbf);
-    
-    # read in information about dbt file
-    if (read (FOXBASEIN, $temp, 32) < 32) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='EOF while reading database header'>\n";
-	}
-	print STDERR "FOXPlugin::read - eof while reading database header\n";
-	close (FOXBASEIN);
-	return -1;
-    }
-    
-    # unpack the header
-    ($dbf{'hasdbt'},
-     $dbf{'modyear'}, $dbf{'modmonth'}, $dbf{'modday'},
-     $dbf{'numrecords'}, $dbf{'headerlen'}, 
-     $dbf{'recordlen'}) = unpack ("CCCCVvv", $temp);
-    
-    # process hasdbt
-    if ($dbf{'hasdbt'} == 131) {
-	$dbf{'hasdbt'} = 1;
-    } elsif ($dbf{'hasdbt'} == 3 || $dbf{'hasdbt'} == 48) {
-	$dbf{'hasdbt'} = 0;
-    } else {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='Does not seem to be a Foxbase file'>\n";
-	}
-	print STDERR "FOXPlugin:read - $filename_full_path doesn't seem to be a Foxbase file\n";
-	return -1;
-    }
-
-    # read in the field description
-    $dbf{'numfields'} = 0;
-    $dbf{'fieldinfo'} = [];
-    while (read (FOXBASEIN, $temp, 1) > 0) {
-	last if ($temp eq "\x0d");
-	last if (read (FOXBASEIN, $temp, 31, 1) < 31);
-
-	my %field = ();
-	$field{'name'} = $self->extracttext($temp, 11);
-	($field{'type'}, $field{'pos'}, $field{'len'}, $field{'dp'}) 
-	    = unpack ("x11a1VCC", $temp);
-
-	push (@{$dbf{'fieldinfo'}}, \%field);
-
-	$dbf{'numfields'} ++;
-    }
-
-    # open the dbt file if we need to
-    my $dbtfullname = $filename_full_path;
-    if ($filename_full_path =~ /f$/) {
-	$dbtfullname =~ s/f$/t/;
-    } else {
-	$dbtfullname =~ s/F$/T/;
-    }
-    if ($dbf{'hasdbt'} && !open (DBTIN, $dbtfullname)) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='Could not read $dbtfullname'>\n";
-	}
-	print STDERR "FOXPlugin::read - couldn't read $dbtfullname\n";
-	close (FOXBASEIN);
-	return -1;
-    }
-
-    # read in and process each record in the database
-    my $numrecords = 0;
-    while (($numrecords < $dbf{'numrecords'}) && 
-	   (read (FOXBASEIN, $temp, $dbf{'recordlen'}) == $dbf{'recordlen'})) {
-
-	# create a new record
-	my $record = [];
-	
-	foreach my $field (@{$dbf{'fieldinfo'}}) {
-	    my $fieldvalue = "";
-	    
-	    if ($field->{'type'} eq "M" && $dbf{'hasdbt'}) {
-		# a memo field, look up this field in the dbt file
-		my $seekpos = substr ($temp, $field->{'pos'}, $field->{'len'});
-
-		$seekpos =~ s/^\s*//;
-		$seekpos = 0 unless $seekpos =~ /^\d+$/;
-		
-		$seekpos = $seekpos * 512;
-
-		if ($seekpos == 0) {
-		    # there is no memo field
-
-		} elsif (seek (DBTIN, $seekpos, 0)) {
-		    while (read (DBTIN, $fieldvalue, 512, length($fieldvalue)) > 0) {
-			last if ($fieldvalue =~ /\cZ/);
-		    }
-
-		    # remove everything after the control-Z
-		    substr($fieldvalue, index($fieldvalue, "\cZ")) = "";
-
-		} else {
-		    print STDERR "\nERROR - seek (to $seekpos) failed\n";
-		}
-
-	    } else {
-		# a normal field
-		$fieldvalue = substr ($temp, $field->{'pos'}, $field->{'len'});
-	    }
-
-	    push (@$record, {%$field, 'value'=>$fieldvalue});
-	}
-
-	# process this record
-	$self->process_record ($pluginfo, $base_dir, $file, $metadata, $processor, 
-			       $numrecords, $record);
-	
-	# finished another record...
-	$numrecords++;
-    }
-
-    # close the dbt file if we need to
-    if ($dbf{'hasdbt'}) {
-	close (DBTIN);
-    }
-
-    # close the dbf file
-    close (FOXBASEIN);
-
-    # finished processing
-    return 1;
-}
-
-
-# will extract a string from some larger string, making it
-# conform to a number of constraints
-sub extracttext {
-    my $self = shift (@_);
-    my ($text, $maxlen, $offset, $stopstr) = @_;
-    $offset = 0 unless defined $offset;
-    $stopstr = "\x00" unless defined $stopstr;
-    
-    # decide where the string finishes
-    my $end = index ($text, $stopstr, $offset);
-    $end = length ($text) if $end < 0;
-    $end = $offset+$maxlen if (defined $maxlen) && ($end-$offset > $maxlen);
-    
-    return "" if ($end <= $offset);
-    return substr ($text, $offset, $end-$offset);
-}
-
-
-# process_record should be overriden for a particular type
-# of database. This default version outputs an html document
-# containing all the fields in the record as a table.
-# It also assumes that the text is in utf-8.
-sub process_record {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $processor, $numrecords, $record) = @_;
-
-    # create a new document
-    my $doc_obj = new doc ($file, "indexed_doc", $self->{'file_rename_method'});
-
-    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-    my $section = $doc_obj->get_top_section();
-
-    $doc_obj->add_metadata($section, "FileFormat", "FOX");
-    $doc_obj->add_metadata($section, "FileSize",   (-s $file));
-
-    # start of document
-    $doc_obj->add_utf8_text($section, "<table>\n");
-
-    # add each field
-    foreach my $field (@$record) {
-	if (defined ($field->{'name'}) && defined ($field->{'value'})) {
-	    $doc_obj->add_utf8_text($section, "  <tr>\n");
-	    $doc_obj->add_utf8_text($section, "    <td>$field->{'name'}</td>\n");
-	    $doc_obj->add_utf8_text($section, "    <td>$field->{'value'}</td>\n");
-	    $doc_obj->add_utf8_text($section, "  </tr>\n");
-	}
-    }
-
-    # end of document
-    $doc_obj->add_utf8_text($section, "</table>\n");
-
-    # add an object id
-    $self->add_OID($doc_obj);
-
-    # process the document
-    $processor->process($doc_obj);
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/LaTeXPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/LaTeXPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,755 +1,0 @@
-###########################################################################
-#
-# LaTeXPlugin.pm
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Written by John McPherson
-# Copyright (C) 2004 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-###########################################################################
-
-# todo:
-#  \includegraphics
-#  parse/remove tex \if ... macros
-
-package LaTeXPlugin;
-
-# System complains about $arguments if the strict is set
-use strict;
-no strict 'refs'; # so we can print to a handle named by a variable
-
-# greenstone packages
-use ReadTextFile;
-use MetadataRead;
-use unicode;
-use util;
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_process_exp() } ];
-
-my $options = { 'name'     => "LaTeXPlugin",
-		'desc'     => "{LaTeXPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub BEGIN {
-    @LaTeXPlugin::ISA = ('MetadataRead', 'ReadTextFile');
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    $self->{'aux_files'} = {};
-    $self->{'dir_num'} = 0;
-    $self->{'file_num'} = 0;
-    return bless $self, $class;
-}
-
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-    return q^\.tex$^;
-}
-
-sub get_default_block_exp {
-    # assume any .eps files are part of the latex stuff
-    return '\.(?:eps)$';
-}
-
-
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my $start=substr($$textref, 0, 200); # first 200 bytes
-
-    if ($start !~ m~\\ (?:documentclass | documentstyle | input | section 
-			| chapter | contents | begin) ~x) {
-	# this doesn't look like latex...
-	return undef;
-    }
-    my $outhandle = $self->{'outhandle'};
-
-    my $cursection = $doc_obj->get_top_section();
-
-    ###### clean up text ######
-    $$textref =~ s/\r$//mg;  # remove dos ^M
-    $$textref =~ s/%.*$//mg; # remove comments
-
-    # convert to utf-8 if not already - assume non ascii => iso-8859-1/latin
-
-    $$textref =~ s@(?<=[[:ascii:]])\xA0+@\xc2\xa0@g; # latin nonbreaking space
-    # check that both sides are ascii, so we don't screw up utf-8 chars
-    $$textref =~ s@ (?<=[[:ascii:]])([\x80-\xff])(?=[[:ascii:]]) @
-	unicode::ascii2utf8($1) @egx; # takes "extended ascii" (ie latin)
-
-
-    ###### find metadata ######
-
-    ## FileFormat metadata ##
-    $doc_obj->add_metadata($cursection, "FileFormat", "LaTeX");
-
-    ### title metadata ###
-    $$textref =~ m@\\title\s*{(.*?)}@s;
-    my $title = $1;
-    if (!$title) {
-	# no title tag. look for a chapter/section heading
-	$$textref =~ m@\\(?:chapter|section)\s*{(.*?)}@s; # will get 1st match
-	$title = $1;
-    }
-    if (!$title) {
-	# no chapter/section heading tags either... use filename
-	$title = $file;
-	$title =~ s/\.tex$//i;
-	$title =~ s/[-_.]/ /g; # turn punctuation into spaces
-    }
-    if ($title) {
-	$title =~ s@\\\\@ @g; # embedded newlines
-	$title = $self->process_latex($title); # no "-html" for title eg in browser
-	$doc_obj->add_utf8_metadata($cursection, "Title", $title);
-    }
-
-    ### creator/individual author metadata ###
-    $$textref =~ m@\\author\s*{((?:{.*?}|.*?)+)}\s*$@sm;
-    my $authors=$1;
-    if ($authors) {
-	# take care of "et al."...
-	$authors =~ s/(\s+et\.?\s+al\.?)\s*$//;
-	my $etal=$1;
-	$etal="" if (!defined ($etal));
-
-	my @authorlist=parse_authors($self, $authors);
-
-	foreach my $author (@authorlist) {
-	    # Add each name to set of Authors
-	    $doc_obj->add_utf8_metadata ($cursection, "Author", $author);
-	}
-
-	# Only want at most one "and" in the Creator field
-	my $creator_str="";
-	if (scalar(@authorlist) > 2) {
-	    my $lastauthor=pop @authorlist;
-	    $creator_str=join(', ', @authorlist);
-	    $creator_str.=" and $lastauthor";
-	} else { # 1 or 2 authors...
-	    $creator_str=join(" and ",@authorlist);
-	}
-	$creator_str.=$etal; # if there was "et al."
-    	$doc_obj->add_utf8_metadata($cursection, "Creator", $creator_str);
-    }
-    ### end of author metadata ###
-
-    ###### process latex for the main text ######
-    $$textref =~ s/^.*?\\begin{document}//s;
-    $$textref =~ s/\\end{document}.*?$//s;
-    $$textref = $self->process_latex("-html",$$textref);
-    $doc_obj->add_utf8_text($cursection, $$textref);
-
-    return 1;
-}
-
-
-# returns a list of author names
-sub parse_authors {
-    my $self=shift;
-    my $authors=shift;
-
-    my $outhandle=$self->{'outhandle'};
-
-    $authors =~ s/\n/ /g; # remove newlines
-	
-    # some people do this for affiliation footnote/dagger
-    $authors =~ s@\$.*?\$@@g; # remove maths from author :(
-
-    # und here for german language...
-    # don't use brackets in pattern, else the matched bit becomes
-    # an element in the list!
-    my @authorlist = split(/\s+and\s+|\s+und\s+/, $authors);
-    my @formattedlist = ();
-    foreach my $author (@authorlist) {
-	$author =~ s/\s*$//; 
-	$author =~ s/^\s*//; 
-	# Reformat and add author name
-	next if $author=~ /^\s*$/;
-
-	# names are "First von Last", "von Last, First"
-	# or "von Last, Jr, First". See the "BibTeXing" manual, page 16
-	my $first="";
-	my $vonlast="";
-	my $jr="";
-	    
-	if ($author =~ /,/) {
-	    my @parts=split(/,\s*/, $author);
-	    $first = pop @parts;
-	    if (scalar(@parts) == 2) {
-		$jr = pop @parts;
-	    }
-	    $vonlast=shift @parts;
-	    if (scalar(@parts) > 0) {
-		print $outhandle $self->{'plugin_type'} .
-		    ": couldn't parse name $author\n";
-		# but we continue anyway...
-	    }
-	} else { # First von Last
-	    my @words = split(/ /, $author);
-	    while (scalar(@words) > 1 && $words[0] !~ /^[a-z]{2..}/) {
-		$first .= " " . shift (@words);
-	    }
-	    $first =~ s/^\s//;
-	    $vonlast = join (' ', @words); # whatever's left...
-	}
-	my $von="";
-	my $last="";
-	if ($vonlast =~ m/^[a-z]/) { # lowercase implies "von"
-	    $vonlast =~ s/^(([a-z]\w+\s+)+)//;
-	    $von = $1;
-	    if (!defined ($von)) {
-		# some non-English names do start with lowercase
-		# eg "Marie desJardins". Also we can get typos...
-		print $outhandle "BibTexPlug: couldn't parse surname $vonlast\n";
-		$von="";
-		if ($vonlast =~ /^[a-z]+$/) {
-		    # if it's all lowercase, uppercase 1st.
-		    $vonlast =~ s/^(.)/\u$1/;
-		}
-	    }
-	    $von =~ s/\s*$//;
-	    $last=$vonlast;
-	} else {
-	    $last=$vonlast;
-	}
-	my $wholename="$first $von $last $jr";
-	$wholename =~ s/ $//; $wholename =~ s/\s+/ /g;
-#	my $fullname = "$last";
-#	$fullname .= " $jr" if ($jr);
-#	$fullname .= ", $first";
-#	$fullname .= " $von" if ($von);
-	push (@formattedlist, $wholename);
-    }
-    return @formattedlist;
-}
-
-
-## following functions based on bibtex plugin ##
-# not actually used at the moment, but might be useful in future?
-sub expand_month {
-    my $text=shift;
-
-    # bibtex style files expand abbreviations for months.
-    # Entries can contain more than one month (eg ' month = jun # "-" # aug, ')
-    $text =~ s/jan/_textmonth01_/g;
-    $text =~ s/feb/_textmonth02_/g;
-    $text =~ s/mar/_textmonth03_/g;
-    $text =~ s/apr/_textmonth04_/g;
-    $text =~ s/may/_textmonth05_/g;
-    $text =~ s/jun/_textmonth06_/g;
-    $text =~ s/jul/_textmonth07_/g;
-    $text =~ s/aug/_textmonth08_/g;
-    $text =~ s/sep/_textmonth09_/g;
-    $text =~ s/oct/_textmonth10_/g;
-    $text =~ s/nov/_textmonth11_/g;
-    $text =~ s/dec/_textmonth12_/g;
-
-    return $text;
-}
-
-
-# If you want basic html formatting (eg \emph -> <em>, \bf, etc), give "-html"
-# as the first argument to this function.
-#
-# Convert accented characters, remove { }, interprete some commands....
-# Note!! This is not comprehensive! Also assumes Latin -> Unicode!
-
-# Also, it sucks quite a bit for complicated/nested commands since it doesn't
-# match { with the corresponding }, only the nearest }
-
-sub process_latex {
-    my $self=shift;
-    my $text=shift;
-
-    my $outhandle=$self->{'outhandle'};
-
-    my $html_markup=0;
-    if ($text =~ /^\-html/) {
-	$html_markup=1;
-	$text=shift;
-    }
-
-    if (! $text) {
-	return $text;
-    }
-    # escape html-sensitive characters
-    $text =~ s@&@&amp;@g;
-    $text =~ s@<@&lt;@g;
-    $text =~ s@>@&gt;@g;
- 
-    # do this before accents, since \= means something different in tabbing
-    # also \> is a tab stop too, and \\ is newline
-    sub do_tabbing {
-	my $tabbing=shift;
-	$tabbing =~ s!^.*\\kill\s*$!!mg; # \kill sets tab stops, kills line
-	$tabbing =~ s~\\(?:=|&gt;)~\xc2\xa0~g; # replace with nbsp
-	$tabbing =~ s~[\\][\\](?:\[.*?\])?\s*$~<br/>~mg;
-	return "<br/>" . $tabbing . "<br/>\n";
-    }
-    $text =~ s@\\begin{tabbing}(.*?)\\end{tabbing}@do_tabbing($1)@ges;
-    sub do_tabular {
-	my $tabular=shift;
-	$tabular =~ s~(?<!\\)\s*&amp;\s*~</td><td>~g;
-	$tabular =~ s~[\\][\\]\s*~</td></tr>\n <tr><td>~g;
-	$tabular =~ s~\\hline~~g; # for now...
-	$tabular =~ s~<td>\s*\\multicolumn{(\d+)}{.*?}~<td colspan="$1">~g;
-	return "<table border=\"1\">\n <tr><td>"
-	    . $tabular . "</td></tr></table>\n";
-    }
-    $text =~ s@\\begin{tabular}(?:\[.*?\])?{.*?}(.*?)\\end{tabular} @
-    		do_tabular($1)  @xges;
-
-    $text =~ s@[\\][\\]\s*\n@ @g; # fold lines ending with \\
-
-    # process maths mode before accents... things like \, mean different!
-    # maths mode
-    $text =~ s@\$\$(.*?)\$\$ 
-	@ process_latex_math($html_markup,$1)
-	@xsge; # multi-line maths: $$ .... $$
-
-    $text =~ s@([^\\])\$(.*?[^\\])\$
-	@$1.process_latex_math($html_markup,$2)@xsge;
-
-
-    # is this an amstext environment, or just custom for that input file?
-    $text =~ s@\\begin{(algorithm)}(.*?)\\end{\1}@remove_equals($2)@ges;
-
-    # convert latex-style accented characters.
-    $self->latex_accents_to_utf8(\$text);
-
-    # replace quotes with utf-8
-
-    $text =~ s/``/\xe2\xc0\x9c/g; # Latex-specific, left-dbl quote (&ldquo;)
-    $text =~ s/''/\xe2\xc0\x9d/g; # Latex-specific, right-dbl quote (&rdquo;)
-    $text =~ s/`/\xe2\xc0\x98/g; # single left quote
-    $text =~ s/'/\xe2\xc0\x99/g; # single right quote
-
-    ###### remove/replace latex commands ######
-    ### commands that expand to something that gets displayed ###
-    $text =~ s~\\ldots~&hellip;~g;
-    $text =~ s~\\hrule~<hr/>\n~g;
-    $text =~ s~\\maketitle~ ~;
-    ### space commands ###
-    $text =~ s~\\[vh]skip\s+\S+~~g;
-    $text =~ s~\\vspace\*?{.*?}~<div>&nbsp;</div>~g; # vertical space
-    $text =~ s~\\\w+skip~ ~g; # \smallskip \medskip \bigskip \baselineskip etc
-    $text =~ s~\\noindent\b~~g;
-    # newpage, etc
-    $text =~ s~\\(?:clearemptydoublepage|newpage)~~g;
-    ### counters, contents, environments, labels, etc ###
-    $text =~ s~\\(?:addcontentsline){.*?}{.*?}{.*}~~g;
-    $text =~ s~\s*\\begin{itemize}\s*~\n<ul>\n~g;
-    $text =~ s~\s*\\end{itemize}\s*~</li></ul>\n~g;
-    $text =~ s~\s*\\begin{enumerate}\s*~<ol>\n~g;
-    $text =~ s~\s*\\end{enumerate}\s*~</li></ol>\n~g;
-    if ($text =~ s~\s*\\item~</li>\n<li>~g) {
-	# (count for first list item)
-	$text =~ s~<([ou])l>\s*</li>\s*~<$1l>~g;
-    }
-    $text =~ s~\\(?:label|begin|end){.*?}\s*\n?~ ~g; # remove tag and contents
-    $text =~ s~\\(?:tableofcontents|listoffigures)~ ~g;
-    ### font sizes/styles ###
-    $text =~ s~\\(?:tiny|small|footnotesize|normalsize|large|Large|huge|Huge)\b~~g;
-
-    if ($html_markup) {
-	$text =~ s~\\section\*?{([^\}]+)}\s*\n?~<H1>$1</H1>\n~g;
-	$text =~ s~\\subsection\*?{(.*?)}\s*\n?~<H2>$1</H2>\n~g;
-	$text =~ s~{\\tt\s*(.*?)}~<tt>$1</tt>~g;
-	$text =~ s~\\(?:texttt|tt|ttseries)\s*{(.*?)}~<tt>$1</tt>~g;
-	$text =~ s~\\emph{(.*?)}~<em>$1</em>~g;
-	$text =~ s~{\\(?:em|it)\s*(.*?)}~<em>$1</em>~g;
-	$text =~ s~{\\(?:bf|bfseries)\s*(.*?)}~<strong>$1</strong>~g;
-	$text =~ s~\\(?:textbf|bf|bfseries)\s*{(.*?)}~<strong>$1</strong>~g;
-    } else {
-	# remove tags for text-only
-	$text =~ s~\\(?:textbf|bf|bfseries|em|emph|tt|rm|texttt)\b~ ~g;
-    }
-    $text =~ s ~ {\\sc\s+(.*?)} ~
-		{<span style="font-variant:\ small-caps">$1</span>} ~gx;
-    # ignore these font tags (if there are any left)
-    # sf is sans-serif
-    $text =~ s~\\(?:mdseries|textmd|bfseries|textbf|sffamily|sf|sc)\b~ ~;
-    #### end font-related stuff ####
-
-    ### remove all other commands with optional arguments... ###
-    # don't remove commands without { }....
-    # $text =~ s~\\\w+(\[.*?\])?\s*~~g;
-    # $text =~ s~\\noopsort{[^}]+\}~~g;
-    # verbatim
-    $text =~ s~\\verb(.)(.*?)\1~verb_text($1)~ge;
-    # remove tags, keep contents for \tag[optional]{contents}
-    while ($text =~ s~\\\w+(\[.*?\])?{([^}]+)}~$2 ~g) {;} # all other commands
-    
-    # remove latex groupings { } (but not \{ or \} )
-    while ($text =~ s/([^\\])[\{\}]/$1/g) {;} # needed for "...}{..."
-    $text =~ s/^\{//; # remove { if first char
-
-    # latex characters
-    # spaces - nobr space (~), opt break (\-), append ("#" - bibtex only)
-    $text =~ s/([^\\])~+/$1 /g; # non-breaking space  "~"
-    # optional break "\-"
-    if ($text =~ m/[^&]\#/) { # concat macros (bibtex) but not HTML codes
-	# the non-macro bits have quotes around them - we just remove quotes
-	# XXX bibtex and latex differ here (for the '#' char)...
-	$text =~ s/([^&])[\"\#]/$1/g;
-    }
-    # dashes. Convert (m|n)-dash into single dash for html.
-    $text =~ s~\-\-+~\-~g;
-
-    # quoted { } chars
-    $text =~ s~\\{~{~g;
-    $text =~ s~\\}~}~g;
-
-    # spaces
-    $text =~ s~\\ ~ ~g;
-
-    # finally to protect against macro language...
-    # greenstone-specific
-    $text =~ s~\[~&\#91;~g;
-    $text =~ s~\]~&\#93;~g;
-    $text =~ s~(?<!\\)([\\_])~\\$1~g;
-
-    if ($html_markup) {
-	$text =~ s~\n{2,}~\n</p>\n<p>~g;
-	return "<p>$text</p>";
-    }
-
-    return $text;
-}
-
-# only used by process_latex for \verb....
-sub verb_text {
-    my $verbatim=shift;
-    $verbatim =~ s/([{}_])/\\$1/g;
-    return $verbatim;
-}
-# only used by process_latex_math
-# returns a unicode char if applicable, otherwise ascii
-sub math_fraction {
-    my $num=$1;
-    my $denom=$2;
-
-    if ($num==1 && $denom==2) {return chr(0xc2).chr(0xbd)}
-    if ($num==1 && $denom==4) {return chr(0xc2).chr(0xbc)}
-    if ($num==3 && $denom==4) {return chr(0xc2).chr(0xbe)}
-    return "$num/$denom";
-}
-
-sub process_latex_math {
-
-    my $text = pop; # if given one or two args, this is the last one...
-    my $html_markup=pop; # if given two args, this is the first one else undef
-
-    $text =~ s~\\,~ ~g; # forces a space?
-    $text =~ s~\\infty~infinity~g;             # or unicode 0x221E...
-
-# use this one when more things can read 3-byte utf8 values like this!
-#    $text =~ s~\\cup\b~\xe2\xc8\xaa~g; # union operator - unicode 0x222a
-    $text =~ s~\\cup\b~ U ~g;
-
-    $text =~ s~\\frac\s*{(.+?)}{(.+?)}~math_fraction($1,$2)~ge;
-
-    if ($html_markup) {
-	$text =~ s~\^{(.*?)}~<sup>$1</sup>~g;  # a^b superscript
-	$text =~ s~\^([^\{])~<sup>$1</sup>~g;
-	$text =~ s~\_{(.*?)}~<sub>$1</sub>~g;  # a_b subscript
-	$text =~ s~\_([^\{])~<sub>$1</sub>~g;
-	
-	$text =~ s~\\ldots~&hellip;~g;         # use html named entity for now
-
-	# put all other command names in italics for now
-	$text =~ s~\\([\w]+)~<i>$1</i> ~g;
-    }
-
-    # special cases, for some input files
-    if ($text =~ m~^\\\w+$~) {
-	$text="{" . $text . "}";
-    }
-
-    return $text;
-}
-
-
-
-sub latex_accents_to_utf8 {
-
-    # note - this is really ugly, but it works. There may be a prettier way
-    # of mapping latex accented chars to utf8, but we just brute force it here.
-    # Also, this isn't complete - not every single possible accented letter
-    # is in here yet, but most of the common ones are.
-
-    my %utf8_chars =
-	(
-	 # acutes
-	 '\'a' => chr(0xc3).chr(0xa1),
-	 '\'c' => chr(0xc4).chr(0x87),
-	 '\'e' => chr(0xc3).chr(0xa9),
-	 '\'i' => chr(0xc3).chr(0xad),
-	 '\'l' => chr(0xc3).chr(0xba),
-	 '\'n' => chr(0xc3).chr(0x84),
-	 '\'o' => chr(0xc3).chr(0xb3),
-	 '\'r' => chr(0xc5).chr(0x95),
-	 '\'s' => chr(0xc5).chr(0x9b),
-	 '\'u' => chr(0xc3).chr(0xba),
-	 '\'y' => chr(0xc3).chr(0xbd),
-	 '\'z' => chr(0xc5).chr(0xba),
-	 # graves
-	 '`a' => chr(0xc3).chr(0xa0),
-	 '`A' => chr(0xc3).chr(0x80),
-	 '`e' => chr(0xc3).chr(0xa8),
-	 '`E' => chr(0xc3).chr(0x88),
-	 '`i' => chr(0xc3).chr(0xac),
-	 '`I' => chr(0xc3).chr(0x8c),
-	 '`o' => chr(0xc3).chr(0xb2),
-	 '`O' => chr(0xc3).chr(0x92),
-	 '`u' => chr(0xc3).chr(0xb9),
-	 '`U' => chr(0xc3).chr(0x99),
-	 # circumflex
-	 '^a' => chr(0xc3).chr(0xa2),
-	 '^A' => chr(0xc3).chr(0x82),
-	 '^c' => chr(0xc4).chr(0x89),
-	 '^C' => chr(0xc4).chr(0x88),
-	 '^e' => chr(0xc3).chr(0xaa),
-	 '^E' => chr(0xc3).chr(0x8a),
-	 '^g' => chr(0xc4).chr(0x9d),
-	 '^G' => chr(0xc4).chr(0x9c),
-	 '^h' => chr(0xc4).chr(0xa5),
-	 '^H' => chr(0xc4).chr(0xa4),
-	 '^i' => chr(0xc3).chr(0xae),
-	 '^I' => chr(0xc3).chr(0x8e),
-	 '^j' => chr(0xc4).chr(0xb5),
-	 '^J' => chr(0xc4).chr(0xb4),
-	 '^o' => chr(0xc3).chr(0xb4),
-	 '^O' => chr(0xc3).chr(0x94),
-	 '^s' => chr(0xc5).chr(0x9d),
-	 '^S' => chr(0xc5).chr(0x9c),
-	 '^u' => chr(0xc3).chr(0xa2),
-	 '^U' => chr(0xc3).chr(0xbb),
-	 '^w' => chr(0xc5).chr(0xb5),
-	 '^W' => chr(0xc5).chr(0xb4),
-	 '^y' => chr(0xc5).chr(0xb7),
-	 '^Y' => chr(0xc5).chr(0xb6),
-	 
-	 # diaeresis
-	 '"a' => chr(0xc3).chr(0xa4),
-	 '"A' => chr(0xc3).chr(0x84),
-	 '"e' => chr(0xc3).chr(0xab),
-	 '"E' => chr(0xc3).chr(0x8b),
-	 '"i' => chr(0xc3).chr(0xaf),
-	 '"I' => chr(0xc3).chr(0x8f),
-	 '"\\\\i' => chr(0xc3).chr(0xaf),
-	 '"\\\\I' => chr(0xc3).chr(0x8f),
-	 '"o' => chr(0xc3).chr(0xb6),
-	 '"O' => chr(0xc3).chr(0x96),
-	 '"u' => chr(0xc3).chr(0xbc),
-	 '"U' => chr(0xc3).chr(0x9c),
-	 '"y' => chr(0xc3).chr(0xbf),
-	 '"Y' => chr(0xc3).chr(0xb8),
-	 # tilde
-	 '~A' => chr(0xc3).chr(0x83),
-	 '~N' => chr(0xc3).chr(0x91),
-	 '~O' => chr(0xc3).chr(0x95),
-	 '~a' => chr(0xc3).chr(0xa3),
-	 '~n' => chr(0xc3).chr(0xb1),
-	 '~o' => chr(0xc3).chr(0xb5),
-	 # caron - handled specially
-	 # double acute
-	 # ring
-	 # dot
-	 '.c' => chr(0xc4).chr(0x8b),
-	 '.C' => chr(0xc4).chr(0x8a),
-	 '.e' => chr(0xc4).chr(0x97),
-	 '.E' => chr(0xc4).chr(0x96),
-	 '.g' => chr(0xc4).chr(0xa1),
-	 '.G' => chr(0xc4).chr(0xa0),
-	 '.I' => chr(0xc4).chr(0xb0),
-	 '.z' => chr(0xc5).chr(0xbc),
-	 '.Z' => chr(0xc5).chr(0xbb),
-	 # macron
-	 '=a' => chr(0xc4).chr(0x81),
-	 '=A' => chr(0xc4).chr(0x80),
-	 '=e' => chr(0xc4).chr(0x93),
-	 '=E' => chr(0xc4).chr(0x92),
-	 '=i' => chr(0xc4).chr(0xab),
-	 '=I' => chr(0xc4).chr(0xaa),
-	 '=o' => chr(0xc4).chr(0x8d),
-	 '=O' => chr(0xc4).chr(0x8c),
-	 '=u' => chr(0xc4).chr(0xab),
-	 '=U' => chr(0xc4).chr(0xaa),
-	 
-	 # stroke - handled specially - see below
-	 
-	 # cedilla - handled specially
-	 );
-    
-# these are one letter latex commands - we make sure they're not a longer
-# command name. eg {\d} is d+stroke, so careful of \d
-    my %special_utf8_chars = 
-	(
-	 # breve
-	 'u g' => chr(0xc4).chr(0x9f),
-	 'u G' => chr(0xc4).chr(0x9e),
-	 'u i' => chr(0xc4).chr(0xad),
-	 'u I' => chr(0xc4).chr(0xac),
-	 'u o' => chr(0xc5).chr(0x8f),
-	 'u O' => chr(0xc5).chr(0x8e),
-	 'u u' => chr(0xc5).chr(0xad),
-	 'u U' => chr(0xc5).chr(0xac),
-	 'u z' => chr(0xc5).chr(0xbe), # !!! no such char, but common mistake
-	 'u Z' => chr(0xc5).chr(0xbd), # used instead of v Z !!!
-	 # caron
-	 'v c' => chr(0xc4).chr(0x8d),
-	 'v C' => chr(0xc4).chr(0x8c),
-	 'v n' => chr(0xc5).chr(0x88),
-	 'v N' => chr(0xc5).chr(0x87),
-	 'v s' => chr(0xc5).chr(0xa1),
-	 'v S' => chr(0xc5).chr(0xa5),
-	 'v z' => chr(0xc5).chr(0xbe),
-	 'v Z' => chr(0xc5).chr(0xbd),
-	 # cedilla
-	 'c c' => chr(0xc3).chr(0xa7),
-	 'c C' => chr(0xc3).chr(0x87),
-	 'c g' => chr(0xc4).chr(0xa3),
-	 'c G' => chr(0xc4).chr(0xa2),
-	 'c k' => chr(0xc4).chr(0xb7),
-	 'c K' => chr(0xc4).chr(0xb6),
-	 'c l' => chr(0xc4).chr(0xbc),
-	 'c L' => chr(0xc4).chr(0xbb),
-	 'c n' => chr(0xc5).chr(0x86),
-	 'c N' => chr(0xc5).chr(0x85),
-	 'c r' => chr(0xc5).chr(0x97),
-	 'c R' => chr(0xc5).chr(0x96),
-	 'c s' => chr(0xc5).chr(0x9f),
-	 'c S' => chr(0xc5).chr(0x9e),
-	 'c t' => chr(0xc5).chr(0xa3),
-	 'c T' => chr(0xc5).chr(0xa2),
-	 # double acute / Hungarian accent
-	 'H O' => chr(0xc5).chr(0x90),
-	 'H o' => chr(0xc5).chr(0x91),
-	 'H U' => chr(0xc5).chr(0xb0),
-	 'H u' => chr(0xc5).chr(0xb1),
-	 
-	 # stroke 
-	 'd' => chr(0xc4).chr(0x91),
-	 'D' => chr(0xc4).chr(0x90),
-	 'h' => chr(0xc4).chr(0xa7),
-#	 'H' => chr(0xc4).chr(0xa6), # !! this normally(!!?) means Hung. umlaut
-	 'i' => chr(0xc4).chr(0xb1), # dotless lowercase i
-	 'l' => chr(0xc5).chr(0x82),
-	 'L' => chr(0xc5).chr(0x81),
-	 'o' => chr(0xc3).chr(0xb8),
-	 'O' => chr(0xc3).chr(0x98),
-	 't' => chr(0xc5).chr(0xa7),
-	 'T' => chr(0xc5).chr(0xa6),
-
-         # other special chars
-         'ss' => chr(0xc3).chr(0x9f), # german ss/szlig/sharp s
-         'aa' =>,chr(0xc3).chr(0xa5), # scandanavian/latin a with ring
-	 );
-
-    my $self=shift;
-    my $textref=shift;
-
-    my $outhandle=$self->{'outhandle'};
-    my $text=$$textref;
-    
-    # remove space (if any) between \ and letter to accent (eg {\' a})
-    $text =~ s!(\\[`'="^~\.])\s(\w)\b!$1$2!g;   # for emacs indenting... `]);
-
-    # remove {} around a single character (eg \'{e})
-    $text =~ s!(\\[`'="^~\.]){(\w)}!{$1$2}!g;  # for emacs indenting... `]);
-
-    ## only in bibtex... not in latex proper?!
-    ### \, is another way of doing cedilla \c
-    ##$text =~ s~\\,(.)~\\c $1~g;
-
-    # remove {} around a single character for special 1 letter commands -
-    # need to insert a space. Eg \v{s}  ->  {\v s}
-    $text =~ s~(\\[uvcH]){(\w)}~{$1 $2}~g;
-
-    # only do if the text contains a '\' character. 
-    if ($text =~ m|\\|) {
-	# "normal" accents - ie non-alpha latex tag
-	# xxx used to have ([\w]\b)@ (for word boundary)
-	while ($text =~ m/\\([`'="^~\.])([\w])/) {      # for emacs `])){
-	    my $tex="$1$2"; my $char="$2";
-	    my $replacement=$utf8_chars{$tex};
-	    if (!defined($replacement)) {
-		$text =~ m~(.{20}\\\Q$tex\E.{20})~s;
-		print $outhandle . $self->{'plugin_type'} .
-		    ": Warning: unknown latex accent \"$tex\""
-		    . " in \"$1\"\n";
-		$replacement=$char;
-	    }
-	    $text =~ s/\\\Q$tex/$replacement/g;
-	}
-
-        # where the following letter matters (eg "sm\o rrebr\o d", \ss{})
-        # only do the change if immediately followed by a space, }, {, or \
-	# one letter accents ( + ss / aa)
-        while ($text =~ m~\\([DdhiLlOoTt]|ss|aa)[{}\s\"\\]~) {
-	    my $tex=$1;
-	    my $replacement=$special_utf8_chars{$tex};
-	    if (!defined($replacement)) {
-		$text =~ m~(.{20}\\\Q$tex\E.{20})~s;
-		print $outhandle $self->{'plugin_type'} .
-		    ": Warning: unknown latex accent \"$tex\""
-		    . " in \"$1\"\n";
-		$replacement=$tex;
-	    }
-	    ($text =~ s/{\\$tex}/$replacement/g) or
-		$text =~ s/\\$tex([{}\s\"\\])/$replacement$1/g;
-
-	}
-
-	# one letter latex accent commands that affect following letter
-        while ($text =~ m~\\([uvcH]) ([\w])~) {
-          my $tex="$1 $2"; my $char="$2";
-          my $replacement=$special_utf8_chars{$tex};
-          if (!defined($replacement)) {
-	      $text =~ m~(.{20}\\\Q$tex\E.{20})~s;
-	      print  $outhandle $self->{'plugin_type'} .
-		  ": Warning: unknown latex accent \"$tex\""
-		  . " in \"$1\"\n";
-	      $replacement=$char;
-	  }
-          $text =~ s/\\$tex/$replacement/g;
-        }
-    }
-    $textref=\$text;
-}
-
-
-# modules must return true
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MediainfoOGVPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MediainfoOGVPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,235 +1,0 @@
-###########################################################################
-#
-# MediainfoOGVPlugin.pm -- Plugin for OGV multimedia files
-#
-# A component of the Greenstone digital library software from the New
-# Zealand Digital Library Project at the University of Waikato, New
-# Zealand.
-#
-# Copyright (C) 2010 Arnaud Yvan
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# MediainfoOGVPlugin - a plugin for OGV multimedia files
-# contributed to Greenstone by Arnaud Yvan
-
-# This is a simple Plugin for importing OGV multimedia files.
-# Mediainfo will retrieve the metadata of the file. A fictional document will
-# be created for every such file, and the file itself will be passed
-# to Greenstone as the "associated file" of the document.
-
-# Here's an example where it is useful: I have a collection of
-# ogg movie files with names like conference_20080402.ogv.
-# I add this line to the collection configuration file:
-
-# plugin MediainfoOGVPlugin -process_exp "*.ogv" -assoc_field "movie"
-
-# A document is created for each movie, with the associated movie
-# file's name in the "movie" metadata field.
-
-# The plugin also add some metadata :
-# Duration (in seconds), filesize, title, artist, location, organization,
-# date, contact, copyright.
-
-
-
-package MediainfoOGVPlugin;
-
-use BasePlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-sub BEGIN {
-    @MediainfoOGVPlugin::ISA = ('BasePlugin');
-}
-
-my $arguments  =  
-    [ { 'name' => "process_exp",
-        'desc' => "{BasePlugin.process_exp}",
-        'type' => "regexp",
-        'deft' => &get_default_process_exp(),
-        'reqd' => "no" },
-      { 'name' => "assoc_field",
-	'desc' => "{MediainfoOGVPlugin.assoc_field}",
-	'type' => "string",
-	'deft' => "Movie",
-	'reqd' => "no" },
-      ];
-
-my $options = { 'name'     => "MediainfoOGVPlugin",
-		'desc'     => "{MediainfoOGVPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub init {
-    my $self = shift(@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    $self->BasePlugin::init($verbosity, $outhandle, $failhandle);   
-
-    # Check that Mediainfo is installed and available on the path (except for Windows 95/98)
-    if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
-	my $result = `mediainfo 2>&1`;
-	if ($? == -1 || $? == 256) {  # Linux and Windows return different values for "program not found"
-	    $self->{'mediainfo_not_installed'} = 1;
-	}
-    }
-
-}
-sub get_default_process_exp {
-    return q^(?i)\.ogv$^;
-}
-
-# MediainfoOGVPlugin processing of doc_obj.
-
-sub process {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    #print STDERR "\n\n**** START of processing MediainfoOGVPlugin\n\n";
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-
-    # check the filename is okay - do we need this??
-    if ($filename_full_path eq "" || $filename_no_path eq "") {
-        print $outhandle "MediainfoOGVPlugin: couldn't process \"$filename_no_path\"\n";
-        return undef;
-    }
-
-    # Add the file as an associated file ...
-    my $section = $doc_obj->get_top_section();
-    my $file_format = "ogv";
-    my $mime_type = "video/ogg";
-    my $assoc_field = $self->{'assoc_field'};
-
-    # The assocfilename is the url-encoded version of the utf8 filename
-    my $assoc_file = $doc_obj->get_assocfile_from_sourcefile();
-
-    $doc_obj->associate_file($filename_full_path, $assoc_file, $mime_type, $section);
-    $doc_obj->add_metadata ($section, "FileFormat", $file_format);
-    $doc_obj->add_metadata ($section, "dc.Format", $file_format);
-    $doc_obj->add_metadata ($section, "MimeType", $mime_type);
-    $doc_obj->add_utf8_metadata ($section, $assoc_field, $doc_obj->get_source()); # Source metadata is already in utf8
-
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($section, "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($section, "srclinkFile", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($section, "srcicon", "_iconogg_");
-
-    # we have no text - add dummy text and NoText metadata
-    $self->add_dummy_text($doc_obj, $section);
-
-    if (defined $self->{'mediainfo_not_installed'}) {
-	# can't do any extracted
-	print STDERR "Mediainfo not installed, so can't extract metadata\n";
-	return 1;
-    }
-
-    
-    # Retrieve the file metadata
-    my $command = "mediainfo --inform=\"General;%Duration/String2%|%Duration%|%FileSize/String2%|%Performer%|%Title%|%Recorded_Date%|%Recorded/Location%|%Producer%|%Copyright%|%LICENSE%|%Publisher%\" \"$filename_full_path\"";
-    print $outhandle "$command\n" if ($verbosity > 2);
-    
-    # execute the mediainfo command and store the output of execution in $videodata
-    # my $video_metadata = `$command`; # backticks operator way
-    # Another way to execute the command: better experience with this than with backticks operator above:
-    # use open() to read the output of executing the command (which is piped through to a handle using the | operator)
-    my $video_metadata;
-    if (open(VMIN,"$command|")) {
-        my $line;
-	
-	# we read a line of output at a time
-        while (defined ($line=<VMIN>)) {
-	    #print STDERR "***** line = $line\n";
-	    
-	    $video_metadata .= $line;
-        }
-	
-        close(VMIN);
-    }
-    
-    print $outhandle "$video_metadata\n" if ($verbosity > 2);
-
-    # There are 10 fields separated by |, split these into ordered, individual-named variables
-    my ($FormattedDuration,$Duration,$Filesize,$Artist,$Title,$Date,$Location,$Organization,$Copyright,$License,$Contact) = split(/\|/,$video_metadata);
-    $Duration = int($Duration/1000);
-    $Artist =~ s/\\047/\'/g; # Perl's way of doing: $artist = `echo $artist | sed \"s/\\047/'/g\"`;
-    $Title =~ s/\\047/\'/g; # $title = `echo $title | sed \"s/\\047/'/g\"`;
-
-    print $outhandle "RESULT = $FormattedDuration\n$Duration\n$Filesize\n$Artist\n$Title\n$Date\n$Location\n$Organization\n$Copyright\n$License\n$Contact\n" if ($verbosity > 2);
-    
- 
-    $doc_obj->add_metadata ($section, "FormattedDuration", $FormattedDuration);
-    $doc_obj->add_metadata ($section, "Duration", $Duration);
- 
-
-    $doc_obj->add_metadata($section,"FileSize",$Filesize);
-    $doc_obj->add_utf8_metadata($section,"dc.Creator",$Artist);
-    $doc_obj->add_utf8_metadata($section,"Artist",$Artist);
-    $doc_obj->add_utf8_metadata($section,"dc.Title",$Title);
-    $doc_obj->add_utf8_metadata($section,"Title",$Title);
-    $doc_obj->add_utf8_metadata($section,"dc.Date",$Date);
-    $doc_obj->add_utf8_metadata($section,"Date",$Date);
-    $doc_obj->add_utf8_metadata($section,"dc.Coverage",$Location);
-    $doc_obj->add_utf8_metadata($section,"Location",$Location);
-    $doc_obj->add_utf8_metadata($section,"dc.Publisher",$Organization);
-    $doc_obj->add_utf8_metadata($section,"Organization",$Organization);
-    $doc_obj->add_utf8_metadata($section,"dc.Rights",$Copyright);
-    $doc_obj->add_utf8_metadata($section,"Copyright",$Copyright);
-    $doc_obj->add_utf8_metadata($section,"dc.accessRights",$License);
-    $doc_obj->add_utf8_metadata($section,"License",$License);
-    $doc_obj->add_utf8_metadata($section,"dc.Contributor",$Contact);
-    $doc_obj->add_utf8_metadata($section,"Contact",$Contact);
- 
-
-    #print STDERR "\n\n**** END of MediainfoOGVPlugin\n\n";
-    
-    return 1;
-}
-
-
-1;
-
-
-
-
-
-
-
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/MetadataRead.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/MetadataRead.pm	(revision 26939)
+++ 	(revision )
@@ -1,83 +1,0 @@
-###########################################################################
-#
-# MetadataRead - like a Java interface that defines that a subclass is
-# a Plugin that extracts Metadata
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package MetadataRead;
-
-use PrintInfo;
-use strict;
-
-# MetadataRead is an abstract superclass that does not inherit from anything else.
-# It exists solely to define the can_process_this_file_for_metadata() method in
-# such a way that those MetadataPlugins that inherit from MetadataRead don't need 
-# to define this method and will always process the files associated with them for 
-# metadata and other plugins in the pipeline won't be passed these files anymore.
-
-# MetadataRead defines method can_process_this_file_for_metadata() with identical
-# signature to BasePlugin. (MetadataRead doesn't inherit from BasePlugin, so it's
-# not 'overriding' it.) Subclasses of MetadataRead that want to use this method 
-# definition can override their inherited BasePlugin version of the method by 
-# listing MetadataRead as the *first* superclass they inherit from in the ISA list.
-# This is the way Perl resolves conflicting method definitions.
-
-my $arguments = [];
-
-my $options = { 'name'     => "MetadataRead",
-		'desc'     => "{MetadataRead.desc}",
-		'abstract' => "yes",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists,$auxiliary) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-	# Like PrintInfo, MetadataRead has no superclass,
-	# so $self is intialised to an empty array.
-	my $self = {};
-    return bless $self, $class;
-
-}
-
-# MetadataPlugins that inherit from MetadataRead will by default
-# process all the metadata in files whose extensions match. 
-# Override this method in a subclass to return undef if other 
-# files should also be allowed to process the metadata therafter.
-sub can_process_this_file_for_metadata {
-    my $self = shift(@_);
-
-#	print STDERR "********* MetadataRead::can_process_this_file_for_metadata() called.\n";
-	
-    return $self->can_process_this_file(@_);
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/OpenDocumentPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/OpenDocumentPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,315 +1,0 @@
-###########################################################################
-#
-# OpenDocumentPlugin.pm -- The Open Document plugin
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Processes OASIS Open Document format.
-# Word processing document: .odt, template: .ott 
-# Spreadsheet document: .ods, template: .ots
-# Presentation document: .odp, template: .otp
-# Graphics document: .odg, template: .otg
-# Formulas document: .odf, template: .otf (not supported)
-
-#This basically extracts any text out of the document, but not much else. 
-
-# this inherits ReadXMLFile, and therefore offers -xslt option, but does
-# nothing with it.
-
-package OpenDocumentPlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use ReadXMLFile;
-use XML::XPath;
-use XML::XPath::XMLParser;
-use Cwd;
-use util;
-use ghtml;
-
-sub BEGIN {
-    @OpenDocumentPlugin::ISA = ('ReadXMLFile');
-}
-
-our @filesProcess = ( "content.xml" , "meta.xml" );
-
-my $arguments = [
-		 { 'name' => "process_exp",
-		   'desc' => "{BasePlugin.process_exp}",
-		   'type' => "regexp",
-		   'deft' =>  &get_default_process_exp() }
-		 ];
-
-my $options = { 'name'     => "OpenDocumentPlugin",
-		'desc'     => "{OpenDocumentPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments};
-
-sub get_default_process_exp { return q^(?i)\.o(?:d|t)(?:t|s|p|g)$^; }
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    $self->{'section'} = "";
-    $self->{'office:meta'} = "";
-    
-    return bless $self, $class;
-}
-
-# want to use BasePlugin's version of this, not ReadXMLFile's
-sub can_process_this_file {
-    my $self = shift(@_);
-    
-    return $self->BasePlugin::can_process_this_file(@_);
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "manifest:manifest";
-}
-
-sub xml_doctype {
-    my $self = shift(@_);
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-    die "The only valid doctype is manifest, $name is not valid" if ($name ne "manifest:manifest");
-}
-
-# Called for every start tag. The $_ variable will contain a copy of the
-# tag and the %_ variable will contain the element's attributes.
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-    my %atts  = %_;
-    $self->{'office:meta'} = $element if $self->{'office:meta'} eq "Start";
-    if($element eq 'office:text') {
-	$self->{'collectedText'} = "";
-    }elsif($element eq 'office:meta') {
-	$self->{'collectedText'} = "";
-	$self->{'office:meta'} = "Start";
-    }elsif($element eq 'meta:document-statistic'){
-	foreach my $att (keys %atts) {
-	    $self->{'doc_obj'}->add_utf8_metadata("",$att,$atts{$att});
-	}
-	
-    }
-}
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-    
-    if($element eq 'office:text') {
-	$self->{'doc_obj'}->add_utf8_text("",$self->{'collectedText'});
-	$self->{'collectedText'} = "";
-    }elsif($element eq $self->{'office:meta'}) {
-	if( $self->{'collectedText'} ne "") {
-	    $self->{'doc_obj'}->add_utf8_metadata("",$self->{'office:meta'},$self->{'collectedText'});
-	    $self->{'doc_obj'}->add_utf8_metadata("","Title",$self->{'collectedText'}) if $self->{'office:meta'} =~ m/:title$/;
-	    $self->{'doc_obj'}->add_utf8_metadata("","Language",$self->{'collectedText'}) if $self->{'office:meta'} =~ m/:language$/;
-	    $self->{'doc_obj'}->add_utf8_metadata("","GENERATOR",$self->{'collectedText'}) if $self->{'office:meta'} =~ m/:generator$/;
-	    
-    	}
-	$self->{'collectedText'} = "";
-        $self->{'office:meta'} = "Start";
-    }elsif($element eq 'office:meta'){
-	$self->{'office:meta'} = "";
-    }elsif($element eq 'office:body'){
-	#some documents have text in other places that should probably be indexed if we can't find any doc text
-	
-	if( defined $self->{'collectedText'} && $self->{'collectedText'} ne "" && $self->{'doc_obj'}->get_text("") eq "") {
-	    $self->{'doc_obj'}->add_utf8_text("",$self->{'collectedText'});
-	}
-    }
-}
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_;
-    if($_ =~ m/\w/i) {
-	$self->{'collectedText'} .= "<br/>" if $self->{'collectedText'} ne "";
-	$self->{'collectedText'} .= "$_";
-    }
-}
-
-#trap start and end document so we do not get our doc_obj closed too soon
-sub xml_start_document {}
-sub xml_end_document {}
-
-sub read {
-    my $self = shift (@_);  
-   
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-
-    my $outhandle = $self->{'outhandle'};
-    # Report that we're processing the file
-    print STDERR "<Processing n='$file' p='OpenDocumentPlugin'>\n" if ($gli);
-    print $outhandle "OpenDocumentPlugin: processing $file\n"
-  	if ($self->{'verbosity'}) > 1;
-
-    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
-    $self->{'file'} = $file;
-    $self->{'filename'} = $filename_full_path;
-    $self->{'filename_no_path'} = $filename_no_path;
-    $self->{'processor'} = $processor;
-    $self->{'metadata'} = $metadata;
-    
-    eval{
-	my ($file_only) = $file =~ /([^\\\/]*)$/;
-	my $tmpdir = &util::get_tmp_filename ();
-	&util::mk_all_dir ($tmpdir);
-	
-	$self->open_document();
-	
-	# save current working directory
-	my $cwd = getcwd();
-	chdir ($tmpdir) || die "Unable to change to $tmpdir";
-	&util::cp ($filename_full_path, $tmpdir);
-	
-	$self->unzip ("\"$file_only\"");
-	foreach my $xmlFile (@OpenDocumentPlugin::filesProcess) {
-	    if (-e $xmlFile) {
-		$self->{'parser'}->parsefile($xmlFile);
-	    }
-	}
-	$self->close_document($filename_full_path,$file_only);
-	
-	chdir ($cwd) || die "Unable to change back to $cwd";
-	&util::rm_r ($tmpdir);
-	
-    };
-    
-    if ($@) {
-
-	# parsefile may either croak somewhere in XML::Parser (e.g. because
-	# the document is not well formed) or die somewhere in ReadXMLFile or a
-	# derived plugin (e.g. because we're attempting to process a
-	# document whose DOCTYPE is not meant for this plugin). For the
-	# first case we'll print a warning and continue, for the second
-	# we'll just continue quietly
-
-	print STDERR "**** Error is: $@\n";
-
-	my ($msg) = $@ =~ /Carp::croak\(\'(.*?)\'\)/;
-	if (defined $msg) {	
-	    my $plugin_name = ref ($self);
-	    print $outhandle "$plugin_name failed to process $file ($msg)\n";
-	}
-
-	# reset ourself for the next document
-	print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-	return -1; # error during processing
-    }
-
-    return 1;
-}
-
-sub unzip {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    system ("unzip $file");
-    &util::rm ($file) if -e $file;
-}
-
-sub close_document() {
-    my $self = shift(@_);
-    my ($filename,$file_only) = @_;
-    
-    my $doc_obj = $self->{'doc_obj'};  
-    my $mimetype = $self->get_mimetype();
-
-    $file_only = $doc_obj->get_assocfile_from_sourcefile(); # url-encoded filename in archives
-    $doc_obj->associate_file($filename, $file_only, $mimetype, "");
-    $doc_obj->associate_file("Thumbnails/thumbnail.png", "thumbnail.png", "image/png", "");
-    my $doc_ext = $filename;
-    $doc_ext =~ s/.*\.od(.)/od$1/;
-    
-    # We use set instead of add here because we only want one value
-    $doc_obj->set_utf8_metadata_element("", "FileFormat", "Open Document");
-
-    #setup to doclink thingi
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ("", "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ("", "srclinkFile", $doc_obj->get_sourcefile());
-    $doc_obj->add_utf8_metadata ("", "srcicon",  "<img border=\"0\" align=\"absmiddle\" src=\"_httpprefix_/collect/[collection]/index/assoc/[assocfilepath]/thumbnail.png\" alt=\"View the Open document\" title=\"View the Open document\">"); 
-
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file_only,$self->{'metadata'},$plugin_filename_encoding);
-
-    $self->set_Source_metadata($doc_obj, $filename, $filename_encoding);
-    $doc_obj->set_utf8_metadata_element("", "FileSize", (-s $filename));
-     
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->extra_metadata ($doc_obj, 
-			   $doc_obj->get_top_section(), 
-			   $self->{'metadata'});
-    
-    # add a Title if none has been found yet
-    $self->title_fallback($doc_obj,"",$file_only);
-   
-    # add an OID
-    $self->add_OID($doc_obj);
-    
-    $doc_obj->add_utf8_metadata("", "Plugin", "$self->{'plugin_type'}");
-
-    # process the document
-    $self->{'processor'}->process($doc_obj);
-    
-    $self->{'num_processed'} ++;
-    return 1;
-}
-
-sub get_mimetype(){
-    my $filename = "mimetype";
-    if (!open (FILEIN,"<$filename")){
-   	print STDERR "Warning: unable to open the $filename\n";
-	return "Unknown OpenDocument Format";
-    }
-    else {
-	my $text = "";
-	while (defined (my $line = <FILEIN>)) {
-	   $text .= $line;   
-	}
-	close(FILEIN);
-	return $text;
-    }
-}
-1;
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/PowerPointPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/PowerPointPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,478 +1,0 @@
-###########################################################################
-#
-# PowerPointPlugin.pm -- plugin for importing Microsoft PowerPoint files.
-#  (basic version supports versions 95 and 97)
-#  (through OpenOffice extension, supports all contemporary formats)
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2002 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package PowerPointPlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-use gsprintf 'gsprintf';
-
-use AutoLoadConverters;
-use ConvertBinaryFile;
-
-sub BEGIN {
-    @PowerPointPlugin::ISA = ('ConvertBinaryFile', 'AutoLoadConverters');
-}
-
-my $openoffice_available = 0;
-
-my $windows_convert_to_list =
-    [ {	'name' => "auto",
-	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-      {	'name' => "html",
-	'desc' => "{ConvertBinaryFile.convert_to.html}" },
-      {	'name' => "text",
-	'desc' => "{ConvertBinaryFile.convert_to.text}" },
-      { 'name' => "pagedimg_jpg",
-	'desc' => "{PowerPointPlugin.convert_to.pagedimg_jpg}" },
-      { 'name' => "pagedimg_gif",
-	'desc' => "{PowerPointPlugin.convert_to.pagedimg_gif}" },
-      { 'name' => "pagedimg_png",
-	'desc' => "{PowerPointPlugin.convert_to.pagedimg_png}" }
-      ];
-
-my $openoffice_convert_to_list = 
-    [ {	'name' => "auto",
-	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
-      {	'name' => "html_multi",
-	'desc' => "{PowerPointPlugin.convert_to.html_multi}" },
-      {	'name' => "text",
-	'desc' => "{ConvertBinaryFile.convert_to.text}" },
-      { 'name' => "pagedimg",
-	'desc' => "{PowerPointPlugin.convert_to.pagedimg}" }
-      ];
-
-my $openoffice_extra_convert_to_list = 
-    [ {	'name' => "html_multi",
-	'desc' => "{PowerPointPlugin.convert_to.html_multi}" },
-      { 'name' => "pagedimg",
-	'desc' => "{PowerPointPlugin.convert_to.pagedimg}" }
-      ];
-
-my $arguments = 
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => "&get_default_process_exp()",  # delayed (see below)
-	}
-      ];
-
-my $opt_windows_args = 
-    [ { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $windows_convert_to_list, 
-	'deft' => "html" },
-      { 'name' => "windows_scripting",
-	'desc' => "{PowerPointPlugin.windows_scripting}",
-	'type' => "flag",
-	'reqd' => "no" }
-      ];
-
-my $opt_office_args = 
-    [ { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $openoffice_convert_to_list, 
-	'deft' => "html" }
-      ];
-
-my $options = { 'name'     => "PowerPointPlugin",
-		'desc'     => "{PowerPointPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in PPT format can be replaced with GS-generated html
-	        'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    # this bit needs to happen later after the arguments array has been 
-    # finished - used for parsing the input args.
-    # push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    # this one needs to go in first, to get the print info in the right order
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $auto_converter_self = new AutoLoadConverters($pluginlist,$inputargs,$hashArgOptLists,["OpenOfficeConverter"],1);
-
-    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
- 	if ($auto_converter_self->{'openoffice_available'}) {
-	    # add openoffice convert_to options into list
-	    push (@$windows_convert_to_list, @$openoffice_extra_convert_to_list);
-	}
-	push(@$arguments,@$opt_windows_args);
-    }
-    elsif ($auto_converter_self->{'openoffice_available'}) {
-	push (@$arguments,@$opt_office_args);
-	$openoffice_available = 1;
-    }
-    # TODO need to do the case where they are both enabled!!! what will the convert to list be???
-
-    # evaluate the default for process_exp  - it needs to be delayed till here so we know if openoffice is available or not. But needs to be done before parsing the args.
-    foreach my $a (@$arguments) {
-	if ($a->{'name'} eq "process_exp") {
-	    my $eval_expr = $a->{'deft'};
-	    $a->{'deft'} = eval "$eval_expr";
-	    last;
-	}
-    }
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-
-    my $cbf_self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-    my $self = BasePlugin::merge_inheritance($auto_converter_self, $cbf_self);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    $self = bless $self, $class;
-    $self->{'file_type'} = "PPT";
-
-    if ($self->{'convert_to'} eq "auto") {
-	if ($self->{'windows_scripting'}) {
-	    $self->{'convert_to'} = "pagedimg_jpg";
-	}
-	else {
-	    $self->{'convert_to'} = "html";
-	}
-    }
-
-   my $outhandle = $self->{'outhandle'};
-
-    # can't have windows_scripting and openoffice_conversion at the same time
-    if ($self->{'windows_scripting'} && $self->{'openoffice_conversion'}) {
-	print $outhandle "Warning: Cannot have -windows_scripting and -openoffice_conversion\n";
-	print $outhandle "         on at the same time.  Defaulting to -windows_scripting\n";
-	$self->{'openoffice_conversion'} = 0;
-    }
-    
-    #these are passed through to gsConvert.pl by ConvertBinaryFile.pm
-    $self->{'convert_options'} = "-windows_scripting" if $self->{'windows_scripting'};
-
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
-
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    push(@$specific_options, "-file_rename_method", "none");
-    push(@$specific_options, "-extract_language") if $self->{'extract_language'};
-
-    if ($secondary_plugin_name eq "HTMLPlugin") {
-	push(@$specific_options, "-processing_tmp_files");
-	push(@$specific_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
-    }
-    elsif ($secondary_plugin_name eq "PagedImagePlugin") {
-	push(@$specific_options, "-processing_tmp_files");
-	#is this true??
-	push(@$specific_options,"-input_encoding", "utf8");
-	if ($self->{'openoffice_conversion'}) {
-	    push(@$specific_options, "-create_thumbnail", "false", "-create_screenview", "false");
-	}
-    }
-
-    $self = bless $self, $class;
-
-    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
-    return $self;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    if ($openoffice_available) {
-	return q^(?i)\.(ppt|pptx|odp)$^;
-    }
-
-    return q^(?i)\.ppt$^;
-}
-
-sub init {
-    my $self = shift (@_);
-
-    # ConvertBinaryFile init
-    $self->SUPER::init(@_);
-    $self->AutoLoadConverters::init(@_);
-
-}
-
-sub begin {
-    my $self = shift (@_);
-
-    $self->AutoLoadConverters::begin(@_);
-    $self->SUPER::begin(@_);
-
-}
-
-sub deinit {
-    my $self = shift (@_);
-    
-    $self->AutoLoadConverters::deinit(@_);
-    $self->SUPER::deinit(@_);
-
-}
-
-# override AutoLoadConverters version, as we need to do more stuff once its converted if we are converting to item file
-sub tmp_area_convert_file {
-    my $self = shift (@_);
-    my ($output_ext, $input_filename, $textref) = @_;
-
-    if ($self->{'openoffice_conversion'}) {
-	if ($self->{'convert_to'} eq "pagedimg") {
-	    $output_ext = "html"; # first convert to html
-	}
-	my ($result, $result_str, $new_filename) = $self->OpenOfficeConverter::convert($input_filename, $output_ext);
-	if ($result == 0) {
-	    my $outhandle=$self->{'outhandle'};
-	    print $outhandle "OpenOfficeConverter Conversion error\n";
-	    print $outhandle $result_str;
-	    return "";
-
-	}
-	#print STDERR "result = $result\n";
-	if ($self->{'convert_to'} eq "pagedimg") {
-	    my $item_filename = $self->generate_item_file($new_filename);
-	    return $item_filename;
-	}
-	return $new_filename;
-
-    }
-    else {
-	return $self->ConvertBinaryFile::tmp_area_convert_file(@_);
-    }
-    # get tmp filename
-}
-
-# override default read in some situations, as the conversion of ppt to html results in many files, and we want them all to be processed.
-sub read {
-    my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    return undef unless $self->can_process_this_file($filename_full_path);
-    
-    # we are only doing something special for html_multi
-    if (!($self->{'openoffice_conversion'} && $self->{'convert_to'} eq "html_multi")) {
-	return $self->BasePlugin::read(@_);
-    }
-    my $outhandle = $self->{'outhandle'};
-    print STDERR "<Processing n='$file' p='$self->{'plugin_type'}'>\n" if ($gli);
-    print $outhandle "$self->{'plugin_type'} processing $file\n"
-	    if $self->{'verbosity'} > 1;
-
-    my $conv_filename = $self->tmp_area_convert_file("html", $filename_full_path);
-    if ("$conv_filename" eq "") {return -1;} # had an error, will be passed down pipeline 
-    if (! -e "$conv_filename") {return -1;} 
-
-    my ($tailname, $html_dirname, $suffix)
-	= &File::Basename::fileparse($conv_filename, "\\.[^\\.]+\$");
-
-    my $collect_file = &util::filename_within_collection($filename_full_path);
-    my $dirname_within_collection = &util::filename_within_collection($html_dirname);
-    my $secondary_plugin = $self->{'secondary_plugins'}->{"HTMLPlugin"};
-
-    my @dir;
-    if (!opendir (DIR, $html_dirname)) {
-	print $outhandle "PowerPointPlugin: Couldn't read directory $html_dirname\n";
-	# just process the original file
-	@dir = ("$tailname.$suffix");
-	
-    } else {
-	@dir = readdir (DIR);
-	closedir (DIR);
-    }
-
-    foreach my $file (@dir) {
-	next unless $file =~ /\.html$/;
-	
-	my ($rv, $doc_obj) = 
-	    $secondary_plugin->read_into_doc_obj ($pluginfo,"", &util::filename_cat($html_dirname,$file), $block_hash, {}, $processor, $maxdocs, $total_count, $gli);
-	if ((!defined $rv) || ($rv<1)) {
-	    # wasn't processed
-	    return $rv;
-	}
-
-	# next block copied from ConvertBinaryFile
-	# from here ...
-	# Override previous gsdlsourcefilename set by secondary plugin
-	
-	$doc_obj->set_source_filename ($collect_file, $self->{'file_rename_method'}); 
-	## set_source_filename does not set the doc_obj source_path which is used in archives dbs for incremental
-	# build. so set it manually.
-	$doc_obj->set_source_path($filename_full_path);
-	$doc_obj->set_converted_filename(&util::filename_cat($dirname_within_collection, $file));
-	
-	my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata,$plugin_filename_encoding);
-	$self->set_Source_metadata($doc_obj, $filename_full_path,$filename_encoding);
-        
-	$doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "Plugin", "$self->{'plugin_type'}");
-	$doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FileSize", (-s $filename_full_path));
-
-	
-	my ($tailname, $dirname, $suffix)
-	    = &File::Basename::fileparse($filename_full_path, "\\.[^\\.]+\$");
-	$doc_obj->set_utf8_metadata_element($doc_obj->get_top_section(), "FilenameRoot", $tailname);
-	
-
-	my $topsection = $doc_obj->get_top_section();
-	$self->add_associated_files($doc_obj, $filename_full_path);
-	
-	# extra_metadata is already called by sec plugin in process??
-	$self->extra_metadata($doc_obj, $topsection, $metadata); # do we need this here??
-	# do any automatic metadata extraction
-	$self->auto_extract_metadata ($doc_obj);
-	
-	# have we found a Title??
-	$self->title_fallback($doc_obj,$topsection,$filename_no_path);
-	
-	# use the one generated by HTMLPlugin, otherwise they all end up with same id.
-	#$self->add_OID($doc_obj);
-	# to here...
-
-	# process it
-	$processor->process($doc_obj);
-	undef $doc_obj;
-    }
-    $self->{'num_processed'} ++;
-
-#    my ($process_status,$doc_obj) = $self->read_into_doc_obj(@_);
-    
-#    if ((defined $process_status) && ($process_status == 1)) {
-	
-	# process the document
-#	$processor->process($doc_obj);
-
-#	$self->{'num_processed'} ++;
-#	undef $doc_obj; 
-#    }
-    # delete any temp files that we may have created
-    $self->clean_up_after_doc_obj_processing();
-
-
-    # if process_status == 1, then the file has been processed.
-    return 1;
-
-}
-
-sub generate_item_file
-{
-    my $self = shift(@_);
-    my ($input_filename) = @_;
-    my $outhandle = $self->{'outhandle'};
-    my ($tailname, $dirname, $suffix)
-	= &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
-
-    # find all the files in the directory
-    if (!opendir (DIR, $dirname)) {
-	print $outhandle "PowerPointPlugin: Couldn't read directory $dirname\n";
-	return $input_filename;
-    }
-
-    my @dir = readdir (DIR);
-    closedir (DIR);
-
-    # start the item file
-    my $itemfile_name = &util::filename_cat($dirname, "$tailname.item");
-
-    # encoding specification????
-    if (!open (ITEMFILE, ">$itemfile_name")) {
-	print $outhandle "PowerPOintPlugin: Couldn't open $itemfile_name for writing\n";
-    }
-    print ITEMFILE "<GeneratedBy>PowerPointPlugin\n";
-    # print the first page
-    my @sorted_dir = sort alphanum_sort @dir;
-    for (my $i = 0; $i < scalar(@sorted_dir); $i++) {
-	my $file = $sorted_dir[$i];
-	if ($file =~ /^img(\d+)\.jpg$/) {
-	    my $num = $1;
-	    $self->tidy_up_html(&util::filename_cat($dirname, "text$num.html"));
-	    print ITEMFILE "$num:img$num.jpg:text$num.html:\n";
-	}
-    }
-    close ITEMFILE;
-    return $itemfile_name;
-
-    
-}
-
-# want to sort img1, img2, ...img10, img11 etc.
-sub alphanum_sort {
-    
-    my ($a_txt, $a_num) = $a =~ /^([^\d]*)(\d*)/;
-    my ($b_txt, $b_num) = $b =~ /^([^\d]*)(\d*)/;
-    
-    if ($a_txt ne $b_txt) { return ($a cmp $b) };
-    return ($a_num <=> $b_num);
-}
-
-# Want to remove the line that links to first page, last page, next page, text etc.
-sub tidy_up_html {
-
-    my $self = shift(@_);
-    my ($filename) = @_;
-    return unless (-f $filename);
-    my $backup_filename = "$filename.bak";
-
-    &File::Copy::copy($filename, $backup_filename);
-
-    open (ORIGINAL, $backup_filename) || return;
-    open(HTMLFILE, ">$filename") || return;
-
-    my $line ="";
-    while ($line = <ORIGINAL>) {
-	if ($line =~ /\<body\>/) {
-	    print HTMLFILE $line;
-	    $line = <ORIGINAL>;
-	    next if $line =~ /\<center\>/;
-	}
-	next if $line =~ /First page/;
-	print HTMLFILE ($line);
-    }
-
-    close HTMLFILE;
-    close ORIGINAL;
-}
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/RTFPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/RTFPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,115 +1,0 @@
-###########################################################################
-#
-# RTFPlugin.pm -- plugin for importing Rich Text Format files.
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package RTFPlugin;
-
-use ConvertBinaryFile;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @RTFPlugin::ISA = ('ConvertBinaryFile');
-}
-
-# currently only converts to HTML
-my $convert_to_list =
-    [ {	'name' => "html",
-	'desc' => "{ConvertBinaryFile.convert_to.html}" } ];
-
-my $arguments =
-    [ { 'name' => "convert_to",
-	'desc' => "{ConvertBinaryFile.convert_to}",
-	'type' => "enum",
-	'reqd' => "yes",
-	'list' => $convert_to_list, 
-	'deft' => "html" },
-      { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "description_tags",
-	'desc' => "{HTMLPlugin.description_tags}",
-	'type' => "flag" }
-];
-
-my $options = { 'name'     => "RTFPlugin",
-		'desc'     => "{RTFPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in rtf can be replaced with GS-generated html
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
- 
-    my $self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    $self->{'file_type'} = "RTF";
-
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-    
-    push(@$specific_options, "-file_rename_method", "none");
-    push(@$specific_options, "-extract_language") if $self->{'extract_language'};
-    if ($secondary_plugin_name eq "TextPlugin") {
-	push(@$specific_options, "-input_encoding", "utf8");
-    }
-    elsif ($secondary_plugin_name eq "HTMLPlugin") {
-	push(@$specific_options, "-description_tags") if $self->{'description_tags'};
-	push(@$specific_options, "-processing_tmp_files");
-    }
-
-    $self = bless $self, $class;
-
-    $self->load_secondary_plugins($class,$secondary_plugin_options, $hashArgOptLists);
-
-    return $self;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-    return q^(?i)\.rtf$^;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/RealMediaPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/RealMediaPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,122 +1,0 @@
-###########################################################################
-#
-# RealMediaPlugin.pm -- Extract metadata from Real Media files
-#
-# Original code by Xin Gao
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package RealMediaPlugin;
-
-
-use BasePlugin;
-use rm::Header::PurePerl;
-
-use strict;
-no strict 'refs'; # make an exception so we can use variables as filehandles
-no strict 'subs';
-
-sub BEGIN {
-    @RealMediaPlugin::ISA = ('BasePlugin');
-}
-
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "RealMediaPlugin",
-		'desc'     => "{RealMediaPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-
-# This plugin processes Real Media files with the suffixes ".rm" and ".rmvb"
-sub get_default_process_exp
-{
-    return q^(?i)(\.rm|rmvb)$^;
-}
-
-
-sub new
-{
-    my ($class) = shift(@_);
-    my ($pluginlist, $inputargs, $hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}}, @{$arguments}); 
-    push(@{$hashArgOptLists->{"OptList"}}, $options); 
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-    
-    return bless $self, $class;
-}
-
-# we don't want to hash on the file
-sub get_oid_hash_type {
-    my $self = shift (@_);
-    return "hash_on_ga_xml";
-}
-
-sub process
-{
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    my $top_section = $doc_obj->get_top_section();
-
-    my $text = "";
-    my $real_media = rm::Header::PurePerl->new($filename_full_path);
-    if (!defined $real_media || !defined $real_media->info) {
-	my $outhandle = $self->{'outhandle'};
-	print $outhandle "RealMediaPlugin: $filename_no_path is not a real media file\n";
-	return undef;
-    }
-    foreach my $key (keys %{$real_media->info})
-    {
-	my $value = $real_media->info->{$key};
-	$doc_obj->add_metadata($top_section, $key, $value);
-	$text .= "$key: $value\n";
-    }
-    
-    $doc_obj->add_utf8_text($top_section, "<pre>\n$text\n</pre>");
-    $doc_obj->add_metadata($top_section, "FileFormat", "RealMedia");
-
-    # srclink_file is now deprecated because of the "_" in the metadataname. Use srclinkFile
-    $doc_obj->add_metadata ($top_section, "srclink_file", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata ($top_section, "srclinkFile", $doc_obj->get_sourcefile());
-    $doc_obj->add_metadata($top_section, "srcicon", "_iconrmvideo_");
-
-    # Add the actual filename as it exists on the file system (URL-encoded) as an 
-    # associated file by making sure we undo any escaped URL-encoding there may be
-    my $assoc_file = $doc_obj->get_assocfile_from_sourcefile(); # just url-encoded filename, no path
-    $doc_obj->associate_file($filename_full_path, $assoc_file, "RealMedia", $top_section);
-
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ReferPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ReferPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,309 +1,0 @@
-###########################################################################
-#
-# ReferPlugin.pm - a plugin for bibliography records in Refer format
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright 2000 Gordon W. Paynter
-# Copyright 1999-2000 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# ReferPlugin reads bibliography files in Refer format.
-#
-# by Gordon W. Paynter (gwp@cs.waikato.ac.nz), November 2000
-#
-# Loosely based on hcibib2Plug by Steve Jones (stevej@cs.waikato.ac.nz).
-# Which was based on EMAILPlug by Gordon Paynter (gwp@cs.waikato.ac.nz).
-# Which was based on old versions of HTMLplug and HCIBIBPlugby by Stefan
-# Boddie and others -- it's hard to tell what came from where, now.
-#
-#
-# ReferPlugin creates a document object for every reference in the file.
-# It is a subclass of SplitTextFile, so if there are multiple records, all
-# are read.
-#
-# Document text:
-#   The document text consists of the reference in Refer format
-#
-# Metadata:
-#	$Creator	%A	Author name
-#	$Title		%T	Title of article of book
-#	$Journal	%J	Title of Journal
-#	$Booktitle	%B	Title of book containing the publication
-#	$Report		%R	Type of Report, paper or thesis
-#	$Volume		%V	Volume Number of Journal
-#	$Number		%N	Number of Journal within Volume
-#	$Editor		%E	Editor name
-#	$Pages		%P	Page Number of article
-#	$Publisher	%I	Name of Publisher
-#	$Publisheraddr	%C	Publisher's address
-#	$Date		%D	Date of publication
-#	$Keywords	%K	Keywords associated with publication 
-#	$Abstract	%X	Abstract of publication
-#	$Copyright	%*	Copyright information for the article
-#
-
-package ReferPlugin;
-
-use SplitTextFile;
-use MetadataRead;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-# ReferPlugin is a sub-class of BasePlugin.
-sub BEGIN {
-    @ReferPlugin::ISA = ('MetadataRead', 'SplitTextFile');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "split_exp",
-	'desc' => "{SplitTextFile.split_exp}",
-	'type' => "regexp",
-	'reqd' => "no",
-	'deft' => &get_default_split_exp() } 
-      ];
-
-my $options = { 'name'     => "ReferPlugin",
-		'desc'     => "{ReferPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'explodes' => "yes",
-		'args'     => $arguments };
-
-# This plugin processes files with the suffix ".bib"
-sub get_default_process_exp {
-    return q^(?i)\.bib$^;
-}
-
-# This plugin splits the input text at blank lines
-sub get_default_split_exp {
-    return q^\n\s*\n^;
-}
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new SplitTextFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-# The process function reads a single bibliographic record and stores
-# it as a new document.
-
-sub process {
-    my $self = shift (@_);
-    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # Check that we're dealing with a valid Refer file
-    return undef unless ($$textref =~ /^\s*%/);
-
-    my $cursection = $doc_obj->get_top_section();
-
-    my %field = ('H', 'Header',
-		 'A', 'Creator',
-		 'T', 'Title',
-		 'J', 'Journal',
-		 'B', 'Booktitle',
-		 'R', 'Report',
-		 'V', 'Volume',
-		 'N', 'Number',
-		 'E', 'Editor',
-		 'P', 'Pages',
-		 'I', 'Publisher',
-		 'C', 'PublisherAddress',
-		 'D', 'Date',
-		 'O', 'OtherInformation',
-		 'K', 'Keywords',
-		 'X', 'Abstract',
-		 '*', 'Copyright');
-
-    # Metadata fields 
-    my %metadata;
-    my ($id, $Creator, $Keywords, $text);
-    my @lines = split(/\n+/, $$textref);
-
-    
-    # Read and process each line in the bib file.
-    # Each file consists of a set of metadata items, one to each line
-    # with the Refer key followed by a space then the associated data
-    foreach my $line (@lines) {
-	
-	# Add each line.  Most lines consist of a field identifer and
-	# then data, and we simply store them, though we treat some
-	# of the fields a bit differently.
-
-	$line =~ s/\s+/ /g;
-	$text .= "$line\n";
-	# $ReferFormat .= "$line\n"; # what is this???
-	
-	next unless ($line =~ /^%[A-Z\*]/);
-	$id = substr($line,1,1);
-	$line =~ s/^%. //;
-   	
-	# Add individual authors in "Lastname, Firstname" format.
-	# (The full set of authors will be added below as "Creator".)
-	if ($id eq "A") {
-
-	    # Reformat and add author name
-	    my @words = split(/ /, $line);
-	    my $lastname = pop @words;
-	    my $firstname = join(" ",  @words);
-	    my $fullname = $lastname . ", " . $firstname;
-	    
-	    # Add each name to set of Authors
-	    if ($fullname =~ /\w/) {
-		$fullname = &text_into_html($fullname);
-		$doc_obj->add_metadata ($cursection, "Author", $fullname);
-	    }
-	}
-
-	# Add individual keywords.
-	# (The full set of authors will be added below as "Keywords".)
-	if ($id eq "K") {
-	    my @keywordlist = split(/,/, $line);
-	    foreach my $k (@keywordlist) {
-		$k = lc($k);
-		$k =~ s/\s*$//; 
-		$k =~ s/^\s*//; 
-		if ($k =~ /\w/) {
-		    $k = &text_into_html($k);
-		    $doc_obj->add_metadata ($cursection, "Keyword", $k);
-		}
-	    } 
-	}
-	
-	# Add this line of metadata
-	$metadata{$id} .= "$line\n";
-    }
-
-
-
-    # Add the various field as metadata
-    my ($f, $name, $value);
-    foreach $f (keys %metadata) {
-	
-	next unless (defined $field{$f});
-	next unless (defined $metadata{$f});	
-
-	$name = $field{$f};
-	$value = $metadata{$f};
-
-	# Add the various field as metadata	
-	
-	# The Creator metadata is found by concatenating authors.
-	if ($f eq "A") {
-
-	    my @authorlist = split(/\n/, $value);
-	    my $lastauthor = pop @authorlist;
-	    my $Creator = "";
-	    if (scalar @authorlist) {
-		$Creator = join(", ", @authorlist) . " and $lastauthor";
-	    } else {
-		$Creator = $lastauthor;
-	    }
-
-	    if ($Creator =~ /\w/) {
-		$Creator = &text_into_html($Creator);
-		$doc_obj->add_metadata ($cursection, "Creator", $Creator);
-	    }
-	}
-
-	# The rest are added in a standard way
-	else {
-	    $value = &text_into_html($value);
-	    $doc_obj->add_metadata ($cursection, $name, $value);
-	}
-
-	# Books and Journals are additionally marked for display purposes
-	if ($f eq "B") {
-	    $doc_obj->add_metadata($cursection, "BookConfOnly", 1);
-	} elsif ($f eq "J") {
-	    $doc_obj->add_metadata($cursection, "JournalsOnly", 1); 
-	}
-
-
-    }
-
-    # Add the text in refer format(all fields)
-    if ($text =~ /\w/) {
-	$text = &text_into_html($text);
-	$doc_obj->add_text ($cursection, $text);
-    }
-    # Add FileFormat as the metadata
-    $doc_obj->add_metadata($cursection,"FileFormat","Refer");
-
-    return 1; # processed the file
-}
-
-1;
-#
-# Convert a text string into HTML.
-#
-# The HTML is going to be inserted into a GML file, so 
-# we have to be careful not to use symbols like ">",
-# which ocurs frequently in email messages (and use
-# &gt instead.
-#
-# This function also turns links and email addresses into hyperlinks,
-# and replaces carriage returns with <BR> tags (and multiple carriage
-# returns with <P> tags).
-#
-
-sub text_into_html {
-    my ($text) = @_;
-
-
-    # Convert problem charaters into HTML symbols
-    $text =~ s/&/&amp;/g;
-    $text =~ s/</&lt;/g;
-    $text =~ s/>/&gt;/g;
-    $text =~ s/\"/&quot;/g;
-    $text =~ s/\'/ /g;
-    $text =~ s/\+/ /g;
-    $text =~ s/\(/ /g;
-    $text =~ s/\)/ /g;
-
-    # convert email addresses and URLs into links
-    $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
-    $text =~ s/(http:\/\/[\w\d\.\-]+[\/\w\d\.\-]*)/<a href=\"$1">$1<\/a>/g;
-
-    # Clean up whitespace and convert \n charaters to <BR> or <P>
-    $text =~ s/ +/ /g;
-    $text =~ s/\s*$//; 
-    $text =~ s/^\s*//; 
-    $text =~ s/\n/\n<BR>/g;
-    $text =~ s/<BR>\s*<BR>/<P>/g;
-
-    return $text;
-}
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/WordPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/WordPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,326 +1,0 @@
-###########################################################################
-#
-# WordPlugin.pm -- plugin for importing Microsoft Word documents
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package WordPlugin;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-no strict 'subs';
-
-use gsprintf 'gsprintf';
-
-use AutoLoadConverters;
-use ConvertBinaryFile;
-
-sub BEGIN {
-    @WordPlugin::ISA = ('ConvertBinaryFile', 'AutoLoadConverters');
-}
-
-my $openoffice_available = 0;
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "regexp",
-	'deft' => "&get_default_process_exp()", # delayed (see below)
-	'reqd' => "no" },
-      { 'name' => "description_tags",
-	'desc' => "{HTMLPlugin.description_tags}",
-	'type' => "flag" }
-      ];
-
-
-my $opt_windows_args = [ { 'name' => "windows_scripting",
-			   'desc' => "{WordPlugin.windows_scripting}",
-			   'type' => "flag",
-
-			   'reqd' => "no" } ];
-
-my $opt_office_args = [ { 'name' => "metadata_fields",
-			  'desc' => "{WordPlugin.metadata_fields}",
-			  'type' => "string",
-			  'deft' => "Title" },
-			{ 'name' => "level1_header",
-			  'desc' => "{StructuredHTMLPlugin.level1_header}",
-			  'type' => "regexp",
-			  'reqd' => "no",
-			  'deft' => "" },
-			{ 'name' => "level2_header",
-			  'desc' => "{StructuredHTMLPlugin.level2_header}",
-			  'type' => "regexp",
-			  'reqd' => "no",
-			  'deft' => "" },
-			{ 'name' => "level3_header",
-			  'desc' => "{StructuredHTMLPlugin.level3_header}",
-			  'type' => "regexp",
-			  'reqd' => "no",
-			  'deft' => "" },
-			{ 'name' => "title_header",
-			  'desc' => "{StructuredHTMLPlugin.title_header}",
-			  'type' => "regexp",
-			  'reqd' => "no",
-			  'deft' => "" },
-			{ 'name' => "delete_toc",
-			  'desc' => "{StructuredHTMLPlugin.delete_toc}",
-			  'type' => "flag",
-			  'reqd' => "no" },
-			{ 'name' => "toc_header",
-			  'desc' => "{StructuredHTMLPlugin.toc_header}",
-			  'type' => "regexp",
-			  'reqd' => "no",
-			  'deft' => "" } ];
-
-
-my $options = { 'name'     => "WordPlugin",
-		'desc'     => "{WordPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'srcreplaceable' => "yes", # Source docs in Word can be replaced with GS-generated html
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    # this bit needs to happen later after the arguments array has been 
-    # finished - used for parsing the input args.
-    # push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    # this one needs to go in first, to get the print info in the right order
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $office_capable = 0;
-    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
-	push(@$arguments,@$opt_windows_args);
-	$office_capable = 1;
-    } 
-
-    my $auto_converter_self = new AutoLoadConverters($pluginlist,$inputargs,$hashArgOptLists,["OpenOfficeConverter"],1);
-
-   if ($auto_converter_self->{'openoffice_available'}) {
-	$office_capable = 1;
-	$openoffice_available = 1;
-    } 
-
-    # these office args apply to windows scripting or to openoffice conversion
-    if ($office_capable) {
-	push(@$arguments,@$opt_office_args);
-    }
-    
-    # evaluate the default for process_exp  - it needs to be delayed till here so we know if openoffice is available or not. But needs to be done before parsing the args.
-    foreach my $a (@$arguments) {
-	if ($a->{'name'} eq "process_exp") {
-	    my $eval_expr = $a->{'deft'};
-	    $a->{'deft'} = eval "$eval_expr";
-	    last;
-	}
-    }
-    
-    # have finished modifying our arguments, add them to ArgList
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-
-    my $cbf_self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
-    my $self = BasePlugin::merge_inheritance($auto_converter_self, $cbf_self);
-
-    if ($self->{'info_only'}) {
-	# don't worry about any options etc
-	return bless $self, $class;
-    }
-
-    $self = bless $self, $class;
-    $self->{'file_type'} = "Word";
-
-    my $outhandle = $self->{'outhandle'};
-
-    if ($self->{'windows_scripting'}) {
-	$self->{'convert_options'} = "-windows_scripting";
-	$self->{'office_scripting'} = 1;
-    }    
-    if ($self->{'openoffice_conversion'}) {
-	if ($self->{'windows_scripting'}) {
-	    print $outhandle "Warning: Cannot have -windows_scripting and -openoffice_conversion\n";
-	    print $outhandle "         on at the same time.  Defaulting to -windows_scripting\n";
-	    $self->{'openoffice_conversion'} = 0;
-	}
-	else {
-	    $self->{'office_scripting'} = 1;
-	}
-    }
-
-    # check convert_to
-    if ($self->{'convert_to'} eq "auto") {
-	$self->{'convert_to'} = "html";
-    }
-    # windows or open office scripting, outputs structuredHTML
-    if (defined $self->{'office_scripting'}) {
-	$self->{'convert_to'} = "structuredhtml";
-    } 
-
-    # set convert_to_plugin and convert_to_ext
-    $self->set_standard_convert_settings();
- 
-    my $secondary_plugin_name = $self->{'convert_to_plugin'};
-    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
-
-    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
-	$secondary_plugin_options->{$secondary_plugin_name} = [];
-    }
-    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
-
-    # following title_sub removes "Page 1" and a leading
-    # "1", which is often the page number at the top of the page. Bad Luck
-    # if your document title actually starts with "1 " - is there a better way?
-    push(@$specific_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
-
-    my $associate_tail_re = $self->{'associate_tail_re'};
-    if ((defined $associate_tail_re) && ($associate_tail_re ne "")) {
-	push(@$specific_options, "-associate_tail_re", $associate_tail_re);
-    }
-    push(@$specific_options, "-file_rename_method", "none");
-
-    if ($secondary_plugin_name eq "StructuredHTMLPlugin") {
-	# Instruct HTMLPlugin (when eventually accessed through read_into_doc_obj)
-	# to extract these metadata fields from the HEAD META fields
-	push (@$specific_options, "-metadata_fields","Title,GENERATOR,date,author<Creator>");
-	push (@$specific_options, "-description_tags") if $self->{'office_scripting'}; 
-	push (@$specific_options, "-extract_language") if $self->{'extract_language'};
-	push (@$specific_options, "-delete_toc") if $self->{'delete_toc'};
-	push (@$specific_options, "-toc_header", $self->{'toc_header'}) if $self->{'toc_header'};
-	push (@$specific_options, "-title_header", $self->{'title_header'}) if $self->{'title_header'};
-	push (@$specific_options, "-level1_header", $self->{'level1_header'}) if $self->{'level1_header'};
-	push (@$specific_options, "-level2_header", $self->{'level2_header'})if $self->{'level2_header'};
-	push (@$specific_options, "-level3_header", $self->{'level3_header'}) if $self->{'level3_header'};
-	push (@$specific_options, "-metadata_fields", $self->{'metadata_fields'}) if $self->{'metadata_fields'};
-	push (@$specific_options, "-metadata_field_separator", $self->{'metadata_field_separator'}) if $self->{'metadata_field_separator'};
-	push(@$specific_options, "-processing_tmp_files");
-	
-    }
-	
-    elsif ($secondary_plugin_name eq "HTMLPlugin") {
-	push(@$specific_options, "-processing_tmp_files");
-	push(@$specific_options,"-input_encoding", "utf8");
-	push(@$specific_options,"-extract_language") if $self->{'extract_language'};
-	push(@$specific_options, "-description_tags") if $self->{'description_tags'};
-	# Instruct HTMLPlugin (when eventually accessed through read_into_doc_obj)
-	# to extract these metadata fields from the HEAD META fields
-	push(@$specific_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
-    }
-
-    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
-
-    return $self;
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    if ($openoffice_available) {
-	return q^(?i)\.(doc|dot|docx|odt|wpd)$^;
-    } elsif ($ENV{'GSDLOS'} =~ m/^windows$/i) { 
-		# if OS is windows, can try using docx2html vbs script to see if they have Word 2007
-		# if the user turns windows_scripting on
-		return q^(?i)\.(docx?|dot)$^;
-	}
-    return q^(?i)\.(doc|dot)$^;
-}
-
-sub init {
-    my $self = shift (@_);
-
-    # ConvertBinaryFile init
-    $self->SUPER::init(@_);
-    $self->AutoLoadConverters::init(@_);
-
-}
-
-sub begin {
-    my $self = shift (@_);
-
-    $self->AutoLoadConverters::begin(@_);
-    $self->SUPER::begin(@_);
-
-}
-
-sub deinit {
-    my $self = shift (@_);
-    
-    $self->AutoLoadConverters::deinit(@_);
-    $self->SUPER::deinit(@_);
-
-}
-
-sub tmp_area_convert_file {
-
-    my $self = shift (@_);
-    return $self->AutoLoadConverters::tmp_area_convert_file(@_);
-
-}
-
-
-sub convert_post_process_old
-{
-    my $self = shift (@_);
-    my ($conv_filename) = @_;
-
-    my $outhandle=$self->{'outhandle'};
-     
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
-
-    # read in file ($text will be in utf8)
-    my $text = "";
-    $self->read_file ($conv_filename, $encoding, $language, \$text);
-
-    # turn any high bytes that aren't valid utf-8 into utf-8.
-    #unicode::ensure_utf8(\$text);
-    
-    # Write it out again!
-    #$self->utf8_write_file (\$text, $conv_filename);
-}
-
-# Modified to cache HTML files for efficieny reasons rather
-# than delete all.  HTML is modified not to use IE's VML.
-# VML uses WML files, so these can be deleted.
-sub cleanup_tmp_area {
-    my ($self) = @_;
-    if (defined $self->{'files_dir'}) {
-	my $html_files_dir = $self->{'files_dir'};
-
-	if (opendir(DIN,$html_files_dir)) {
-	    my @wmz_files = grep( /\.wmz$/, readdir(DIN));
-	    foreach my $f (@wmz_files) {
-		my $full_f = &util::filename_cat($html_files_dir,$f);
-		&util::rm($full_f);
-	    }
-	    closedir(DIN);
-	}
-	else {
-	    # if HTML file has no supporting images, then no _files dir made
-	    # => do nothing
-	}
-    }
-}
-
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/ZIPPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/ZIPPlugin.pm	(revision 26939)
+++ 	(revision )
@@ -1,231 +1,0 @@
-###########################################################################
-#
-# ZIPPlugin.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# plugin which handles compressed and/or archived input formats
-#
-# currently handled formats and file extensions are:
-# 
-# gzip (.gz, .z, .tgz, .taz)
-# bzip (.bz)
-# bzip2 (.bz2)
-# zip (.zip .jar)
-# tar (.tar)
-#
-# this plugin relies on the following utilities being present 
-# (if trying to process the corresponding formats)
-#
-# gunzip (for gzip)
-# bunzip (for bzip)
-# bunzip2 
-# unzip (for zip)
-# tar (for tar) 
-
-
-package ZIPPlugin;
-
-use BasePlugin;
-use plugin;
-use util;
-use Cwd;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-BEGIN {
-    @ZIPPlugin::ISA = ('BasePlugin');
-}
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "string",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" } ];
-
-my $options = { 'name'     => "ZIPPlugin",
-		'desc'     => "{ZIPPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    return bless $self, $class;
-}
-
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    # Are we actually incremental and doing import?
-    my $proc_package_name = ref $processor;
-    if ($proc_package_name !~ /buildproc$/ && $self->{'incremental'} == 1) {
-        # Get the infodbtype value for this collection from the arcinfo object
-        my $infodbtype = $processor->getoutputinfo()->{'infodbtype'};
-	$infodbtype = "gdbm" if $infodbtype eq "gdbm-txtgz"; # in archives, cannot use txtgz version
-	my $output_dir = $processor->getoutputdir();
-    	my $archives_inf = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $output_dir);
-
-	if ( -e $archives_inf ) {
-	    $self->{'actually_incremental'} = 1;
-	}
-    }
-   
-    
-}
-# this is a recursive plugin
-sub is_recursive {
-    my $self = shift (@_);
-
-    return 1;
-}
-
-sub get_default_process_exp {
-    return q^(?i)\.(gz|tgz|z|taz|bz|bz2|zip|jar|tar)$^;
-}
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # can we process this file??
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    return undef unless $self->can_process_this_file($filename_full_path);
-	
-    my $tmpdir = $file;
-    $tmpdir =~ s/\.[^\.]*//;
-    $tmpdir = &util::rename_file($tmpdir, $self->{'file_rename_method'});
-    $tmpdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp", $tmpdir);
-    &util::mk_all_dir ($tmpdir);
-    
-    print $outhandle "ZIPPlugin: extracting $filename_no_path to $tmpdir\n"
-	if $self->{'verbosity'} > 1;
-    
-    # save current working directory
-    my $cwd = cwd();
-    chdir ($tmpdir) || die "Unable to change to $tmpdir";
-    &util::cp ($filename_full_path, $tmpdir);
-    
-    if ($file =~ /\.bz$/i) {
-	$self->bunzip ($filename_no_path);
-    } elsif ($file =~ /\.bz2$/i) {
-	$self->bunzip2 ($filename_no_path);
-    } elsif ($file =~ /\.(zip|jar)$/i) {
-	$self->unzip ($filename_no_path);
-    } elsif ($file =~ /\.tar$/i) {
-	$self->untar ($filename_no_path);
-    } else {
-	$self->gunzip ($filename_no_path);
-    }
-    
-    chdir ($cwd) || die "Unable to change back to $cwd";
-    
-    # do the blocking step inside the folder
-    &plugin::file_block_read ($pluginfo, "", $tmpdir,
-			      $block_hash, $metadata, $gli);
-
-    # if we are incremental, then we need to add all the files in the tmp folder into the new_files list otherwise they won't get processed.
-    if ($self->{'actually_incremental'}) {
-	my @file_list = ();
-	&inexport::add_dir_contents_to_list($tmpdir, \@file_list);
-	foreach my $file (@file_list) {
-	    $block_hash->{'new_files'}->{$file} = 1;
-	}
-    }
-    # all files in the tmp folder need to get the gsdlzipfilenmae metadata
-    my $this_metadata = {};
-    $this_metadata->{"gsdlzipfilename"} = $filename_full_path;
-    &metadatautil::combine_metadata_structures($this_metadata, $metadata);
-    my $numdocs = &plugin::read ($pluginfo, "", $tmpdir, $block_hash, $this_metadata, $processor, $maxdocs, $total_count, $gli);
-    &util::rm_r ($tmpdir);
-    
-    $self->{'num_archives'} ++;
-    
-    return $numdocs;
-    
-}
-
-sub bunzip {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    if (system ("bunzip \"$file\"")!=0)
-    {
-	&util::rm ($file);
-    }
-}
-
-sub bunzip2 {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    if (system ("bunzip2 \"$file\"")!=0)
-    {
-	&util::rm ($file);
-    }
-}
-
-sub unzip {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    system ("unzip \"$file\"");
-    &util::rm ($file) if -e $file;
-}
-
-sub untar {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    system ("tar xf \"$file\"");
-    &util::rm ($file) if -e $file;
-}
-
-sub gunzip {
-    my $self = shift (@_);
-    my ($file) = @_;
-
-    if (system ("gunzip \"$file\"")!=0)
-    {
-	&util::rm ($file);
-    };
-}
-
-
-
-1;
