Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm	(revision 26944)
+++ 	(revision )
@@ -1,708 +1,0 @@
-###########################################################################
-#
-# DirectoryPlugin.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# DirectoryPlugin is a plugin which recurses through directories processing
-# each file it finds - which basically means passing it down the plugin 
-# pipeline
-
-package DirectoryPlugin;
-
-use PrintInfo;
-use plugin;
-use util;
-use metadatautil;
-
-use File::Basename;
-use strict;
-no strict 'refs';
-no strict 'subs';
-
-use Encode;
-
-BEGIN {
-    @DirectoryPlugin::ISA = ('PrintInfo');
-}
-
-my $arguments =
-    [ { 'name' => "block_exp",
-	'desc' => "{BasePlugin.block_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_block_exp(),
-	'reqd' => "no" },
-      # this option has been deprecated. leave it here for now so we can warn people not to use it
-      { 'name' => "use_metadata_files",
-	'desc' => "{DirectoryPlugin.use_metadata_files}",
-	'type' => "flag",
-	'reqd' => "no",
-	'hiddengli' => "yes" },
-      { 'name' => "recheck_directories",
-	'desc' => "{DirectoryPlugin.recheck_directories}",
-	'type' => "flag",
-	'reqd' => "no" } ];
-    
-my $options = { 'name'     => "DirectoryPlugin",
-		'desc'     => "{DirectoryPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists);
-    
-    if ($self->{'info_only'}) {
-	# don't worry about any options or initialisations etc
-	return bless $self, $class;
-    }
-
-    # we have left this option in so we can warn people who are still using it
-    if ($self->{'use_metadata_files'}) {
-	die "ERROR: DirectoryPlugin -use_metadata_files option has been deprecated. Please remove the option and add MetadataXMLPlug to your plugin list instead!\n";
-    }
-	
-    $self->{'num_processed'} = 0;
-    $self->{'num_not_processed'} = 0;
-    $self->{'num_blocked'} = 0;
-    $self->{'num_archives'} = 0;
-
-    $self->{'subdir_extrametakeys'} = {};
-
-    return bless $self, $class;
-}
-
-# called once, at the start of processing
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    # verbosity is passed through from the processor
-    $self->{'verbosity'} = $verbosity;
-
-    # as are the outhandle and failhandle
-    $self->{'outhandle'} = $outhandle if defined $outhandle;
-    $self->{'failhandle'} = $failhandle;
-
-}
-
-# called once, after all passes have finished
-sub deinit {
-    my ($self) = @_;
-
-}
-
-# called at the beginning of each plugin pass (import has one, building has many)
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    # Only lookup timestamp info for import.pl, and only if incremental is set
-    my $proc_package_name = ref $processor;
-    if ($proc_package_name !~ /buildproc$/ && $self->{'incremental'} == 1) {
-        # Get the infodbtype value for this collection from the arcinfo object
-        my $infodbtype = $processor->getoutputinfo()->{'infodbtype'};
-	$infodbtype = "gdbm" if $infodbtype eq "gdbm-txtgz"; # in archives, cannot use txtgz version
-	my $output_dir = $processor->getoutputdir();
-    	my $archives_inf = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $output_dir);
-
-	if ( -e $archives_inf ) {
-	    $self->{'inf_timestamp'} = -M $archives_inf;
-	}
-    }
-}
-
-sub remove_all {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-}
-
-
-sub remove_one {
-    my $self = shift (@_);
-    my ($file, $oids, $archivedir) = @_;
-    return undef; # this will never be called for directories (will it??)
-
-}
-
-
-# called at the end of each plugin pass
-sub end {
-    my ($self) = shift (@_);
-
-}
-
-
-
-# return 1 if this class might recurse using $pluginfo
-sub is_recursive {
-    my $self = shift (@_);
-    
-    return 1;
-}
-
-sub get_default_block_exp {
-    my $self = shift (@_);
-    
-    return '(?i)(CVS|\.svn|Thumbs\.db|OIDcount|~)$';
-}
-
-sub check_directory_path {
-
-    my $self = shift(@_);
-    my ($dirname) = @_;
-    
-    return undef unless (-d $dirname);
-
-    return 0 if ($self->{'block_exp'} ne "" && $dirname =~ /$self->{'block_exp'}/);
-
-    my $outhandle = $self->{'outhandle'};
-    
-    # check to make sure we're not reading the archives or index directory
-    my $gsdlhome = quotemeta($ENV{'GSDLHOME'});
-    if ($dirname =~ m/^$gsdlhome\/.*?\/import.*?\/(archives|index)$/) {
-	print $outhandle "DirectoryPlugin: $dirname appears to be a reference to a Greenstone collection, skipping.\n";
-        return 0;
-    }
-    
-    # check to see we haven't got a cyclic path...
-    if ($dirname =~ m%(/.*){,41}%) {
-	print $outhandle "DirectoryPlugin: $dirname is 40 directories deep, is this a recursive path? if not increase constant in DirectoryPlugin.pm.\n";
-	return 0;
-    }
-    
-    # check to see we haven't got a cyclic path...
-    if ($dirname =~ m%.*?import/(.+?)/import/\1.*%) {
-	print $outhandle "DirectoryPlugin: $dirname appears to be in a recursive loop...\n";
-	return 0;
-    }
-
-    return 1;
-}
-
-# this may be called more than once
-sub sort_out_associated_files {
-
-    my $self = shift (@_);
-    my ($block_hash) = @_;
-    if (!scalar (keys %{$block_hash->{'shared_fileroot'}})) {
-	return;
-    }
-
-    $self->{'assocfile_info'} = {} unless defined $self->{'assocfile_info'};
-    my $metadata = $self->{'assocfile_info'};
-    foreach my $prefix (keys %{$block_hash->{'shared_fileroot'}}) {
-	my $record = $block_hash->{'shared_fileroot'}->{$prefix};
-
-	my $tie_to = $record->{'tie_to'};
-	my $exts = $record->{'exts'};
-	
-	if ((defined $tie_to) && (scalar (keys %$exts) > 0)) {
-	    # set up fileblocks and assocfile_tobe
-	    my $base_file = "$prefix$tie_to";
-	    $metadata->{$base_file} = {} unless defined $metadata->{$base_file};
-	    my $base_file_metadata = $metadata->{$base_file};
-	    
-	    $base_file_metadata->{'gsdlassocfile_tobe'} = [] unless defined $base_file_metadata->{'gsdlassocfile_tobe'};
-	    my $assoc_tobe = $base_file_metadata->{'gsdlassocfile_tobe'};
-	    foreach my $e (keys %$exts) {
-		# block the file
-		&util::block_filename($block_hash,"$prefix$e");
-		# set up as an associatd file
-		print STDERR "  $self->{'plugin_type'}: Associating $prefix$e with $tie_to version\n";
-		my $mime_type = ""; # let system auto detect this
-		push(@$assoc_tobe,"$prefix$e:$mime_type:"); 
-
-	    }
-	}
-    } # foreach record
-
-    $block_hash->{'shared_fileroot'} = undef;
-    $block_hash->{'shared_fileroot'} = {};
-
-}
-
-
-# do block exp OR special blocking ???
-
-sub file_is_blocked {
-    my $self = shift (@_);
-    my ($block_hash, $filename_full_path) = @_;
-
-    $filename_full_path = &util::upgrade_if_dos_filename($filename_full_path);
-###    print STDERR "*** DirectoryPlugin::file_is_blocked $filename_full_path\n";
-
-    if ($ENV{'GSDLOS'} =~ m/^windows$/) {
-	# on windows, all block paths are lowercased.
-	my $lower_filename = lc ($filename_full_path);
-	if (defined $block_hash->{'file_blocks'}->{$lower_filename}) {
-	    $self->{'num_blocked'} ++;
-	    return 1;
-	}
-    }
-    else {
-	if (defined $block_hash->{'file_blocks'}->{$filename_full_path}) {
-	    $self->{'num_blocked'} ++;
-	    return 1;
-	}
-    }
-    # check Directory plugin's own block_exp 
-    if ($self->{'block_exp'} ne "" && $filename_full_path =~ /$self->{'block_exp'}/) {
-	$self->{'num_blocked'} ++;
-	return 1; # blocked
-    }
-    return 0;
-}
-
-
-
-sub file_block_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-    
-    # Calculate the directory name and ensure it is a directory and
-    # that it is not explicitly blocked.
-    my $dirname = $file;
-    $dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
-
-    my $directory_ok = $self->check_directory_path($dirname);
-    return $directory_ok unless (defined $directory_ok && $directory_ok == 1);
-
-    print $outhandle "Global file scan checking directory: $dirname\n" if ($verbosity > 0);
-
-    $block_hash->{'all_files'} = {} unless defined $block_hash->{'all_files'};
-    $block_hash->{'metadata_files'} = {} unless defined $block_hash->{'metadata_files'};
-
-    $block_hash->{'file_blocks'} = {} unless defined $block_hash->{'file_blocks'};
-    $block_hash->{'shared_fileroot'} = {} unless defined $block_hash->{'shared_fileroot'};
-
-     # Recur over directory contents.
-    my (@dir, $subfile);
-    #my $count = 0;
-    
-    print $outhandle "DirectoryPlugin block: getting directory $dirname\n" if ($verbosity > 2);
-    
-    # find all the files in the directory
-    if (!opendir (DIR, $dirname)) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
-	}
-	print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
-	return -1; # error in processing
-    }
-    @dir = readdir (DIR);
-    closedir (DIR);
-    
-    for (my $i = 0; $i < scalar(@dir); $i++) {
-	my $raw_subfile = $dir[$i];
-	next if ($raw_subfile =~ m/^\.\.?$/);
-
-	my $this_file_base_dir = $base_dir;
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
-
-	# Recursively read each $raw_subfile
-	print $outhandle "DirectoryPlugin block recurring: $raw_file_subfile\n" if ($verbosity > 2);
-	
-	#$count += &plugin::file_block_read ($pluginfo, $this_file_base_dir,
-
-	&plugin::file_block_read ($pluginfo, $this_file_base_dir,
-				  $raw_file_subfile,
-				  $block_hash, $metadata, $gli);
-	
-    }
-    $self->sort_out_associated_files($block_hash);
-    #return $count;
-	return 1;
-    
-}
-
-# We don't do metadata_read
-sub metadata_read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, 
-	$extrametakeys, $extrametadata, $extrametafile,
-	$processor, $gli, $aux) = @_;
-
-    return undef;
-}
-
-
-# return number of files processed, undef if can't process
-# Note that $base_dir might be "" and that $file might 
-# include directories
-
-# This function passes around metadata hash structures.  Metadata hash
-# structures are hashes that map from a (scalar) key (the metadata element
-# name) to either a scalar metadata value or a reference to an array of
-# such values.
-
-sub read {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $in_metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-    
-    # Calculate the directory name and ensure it is a directory and
-    # that it is not explicitly blocked.
-    my $dirname;
-    if ($file eq "") {
-	$dirname = $base_dir;
-    } else {
-	$dirname = $file;
-	$dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
-    }
-
-    my $directory_ok = $self->check_directory_path($dirname);
-    return $directory_ok unless (defined $directory_ok && $directory_ok == 1);
-        
-    if (($verbosity > 2) && ((scalar keys %$in_metadata) > 0)) {
-        print $outhandle "DirectoryPlugin: metadata passed in: ", 
-	join(", ", keys %$in_metadata), "\n";
-    }
-    
-
-    # Recur over directory contents.
-    my (@dir, $subfile);
-    
-    print $outhandle "DirectoryPlugin read: getting directory $dirname\n" if ($verbosity > 2);
-    
-    # find all the files in the directory
-    if (!opendir (DIR, $dirname)) {
-	if ($gli) {
-	    print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
-	}
-	print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
-	return -1; # error in processing
-    }
-    @dir = readdir (DIR);
-    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dir;
-    closedir (DIR);
-
-    # Re-order the files in the list so any directories ending with .all are moved to the end
-    for (my $i = scalar(@dir) - 1; $i >= 0; $i--) {
-	if (-d &util::filename_cat($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
-	    push(@dir, splice(@dir, $i, 1));
-	}
-    }
-
-    # setup the metadata structures. we do a metadata_read pass to see if there is any additional metadata, then pass it to read
-    
-    my $additionalmetadata = 0;      # is there extra metadata available?
-    my %extrametadata;               # maps from filespec to extra metadata keys
-    my %extrametafile;               # maps from filespec to the metadata.xml (or similar) file it came from
-    my @extrametakeys;               # keys of %extrametadata in order read
-
-
-    my $os_dirsep = &util::get_os_dirsep();
-    my $dirsep    = &util::get_dirsep();
-    my $base_dir_regexp = $base_dir;
-    $base_dir_regexp =~ s/\//$os_dirsep/g;
-    my $local_dirname = $dirname;
-    
-    $local_dirname =~ s/^$base_dir_regexp($os_dirsep)*//;
-    # if we are in import folder, then local_dirname will be empty
-    if ($local_dirname ne "") {
-	# look for extra metadata passed down from higher folders 
-	$local_dirname .= $dirsep;
-	if (defined $self->{'subdir_extrametakeys'}->{$local_dirname}) {
-	    my $extrakeys = $self->{'subdir_extrametakeys'}->{$local_dirname};
-	    foreach my $ek (@$extrakeys) {
-		my $extrakeys_re  = $ek->{'re'};
-		my $extrakeys_md  = $ek->{'md'};
-		my $extrakeys_mf  = $ek->{'mf'};
-		push(@extrametakeys,$extrakeys_re);
-		$extrametadata{$extrakeys_re} = $extrakeys_md;
-		$extrametafile{$extrakeys_re} = $extrakeys_mf;
-	    }
-	    delete($self->{'subdir_extrametakeys'}->{$local_dirname});
-	}
-    }
-    # apply metadata pass for each of the files in the directory -- ignore
-    # maxdocs here
-    my $num_files = scalar(@dir);
-    for (my $i = 0; $i < scalar(@dir); $i++) {
-	my $subfile = $dir[$i];
-	next if ($subfile =~ m/^\.\.?$/);
-
-	my $this_file_base_dir = $base_dir;
-	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
-
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
-	my $raw_full_filename = &util::filename_cat($this_file_base_dir, $raw_file_subfile);
-
-	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
-	    print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for metadata_read\n" if ($verbosity > 2);
-	    next;
-	}
-
-        # If we are processing a manifest file, then we don't have to guess at
-        # what metadata source files to process. Only process those that appear
-        # in the metadata_files list in the block_hash. [jmt12]
-        if (defined $block_hash && defined $block_hash->{'manifest'} && $block_hash->{'manifest'} eq 'true' 
-            && !-d $raw_full_filename
-            && !$block_hash->{'metadata_files'}->{$raw_full_filename})
-        {
-          print $outhandle "**** Not in manifest. Ignoring metadata in $subfile\n" if ($verbosity > 3);
-          next;
-        }
-
-	
-	# Recursively read each $raw_subfile
-	print $outhandle "DirectoryPlugin metadata recurring: $raw_subfile\n" if ($verbosity > 2);
-	
-	&plugin::metadata_read ($pluginfo, $this_file_base_dir,
-				$raw_file_subfile,$block_hash,
-				\@extrametakeys, \%extrametadata,
-				\%extrametafile,
-				$processor, $gli);
-	$additionalmetadata = 1;
-    }
-
-    # filter out any extrametakeys that mention subdirectories and store
-    # for later use (i.e. when that sub-directory is being processed)
-    foreach my $ek (@extrametakeys) {
-	my ($subdir_re,$extrakey_dir) = &File::Basename::fileparse($ek);
-	$extrakey_dir =~ s/\\\./\./g; # remove RE syntax for .
-	$extrakey_dir =~ s/\\\\/\\/g; # remove RE syntax for \
-
-	my $dirsep_re = &util::get_re_dirsep();
-
-	my $ek_non_re = $ek;
-	$ek_non_re =~ s/\\\./\./g; # remove RE syntax for .
-	$ek_non_re =~ s/\\\\/\\/g; # remove RE syntax for \
-	if ($ek_non_re =~ m/$dirsep_re/) { # specifies at least one directory
-	    my $md = $extrametadata{$ek};
-	    my $mf = $extrametafile{$ek};
-
-	    my $subdir_extrametakeys = $self->{'subdir_extrametakeys'};
-
-	    my $subdir_rec = { 're' => $subdir_re, 'md' => $md, 'mf' => $mf };
-
-	    # when its looked up, it must be relative to the base dir
-	    push(@{$subdir_extrametakeys->{"$local_dirname$extrakey_dir"}},$subdir_rec);
-	    #push(@{$subdir_extrametakeys->{"$extrakey_dir"}},$subdir_rec);
-	}
-    }
-    
-    # import each of the files in the directory
-    my $count=0;
-    for (my $i = 0; $i <= scalar(@dir); $i++) {
-	# When every file in the directory has been done, pause for a moment (figuratively!)
-	# If the -recheck_directories argument hasn't been provided, stop now (default)
-	# Otherwise, re-read the contents of the directory to check for new files
-	#   Any new files are added to the @dir list and are processed as normal
-	#   This is necessary when documents to be indexed are specified in bibliographic DBs
-	#   These files are copied/downloaded and stored in a new folder at import time
-	if ($i == $num_files) {
-	    last unless $self->{'recheck_directories'};
-
-	    # Re-read the files in the directory to see if there are any new files
-	    last if (!opendir (DIR, $dirname));
-	    my @dirnow = readdir (DIR);
-	    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dirnow;
-	    closedir (DIR);
-
-	    # We're only interested if there are more files than there were before
-	    last if (scalar(@dirnow) <= scalar(@dir));
-
-	    # Any new files are added to the end of @dir to get processed by the loop
-	    my $j;
-	    foreach my $subfilenow (@dirnow) {
-		for ($j = 0; $j < $num_files; $j++) {
-		    last if ($subfilenow eq $dir[$j]);
-		}
-		if ($j == $num_files) {
-		    # New file
-		    push(@dir, $subfilenow);
-		}
-	    }
-	    # When the new files have been processed, check again
-	    $num_files = scalar(@dir);
-	}
-
-	my $subfile = $dir[$i];
-	last if ($maxdocs != -1 && ($count + $total_count) >= $maxdocs);
-	next if ($subfile =~ /^\.\.?$/);
-
-	my $this_file_base_dir = $base_dir;
-	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
-
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
-	my $raw_full_filename 
-	    = &util::filename_cat($this_file_base_dir,$raw_file_subfile);
-
-	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
-	    print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for read\n" if ($verbosity > 2);
-	    next;
-	}
-	#print STDERR "processing $raw_full_filename\n";
-	# Follow Windows shortcuts
-	if ($raw_subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
-	    require Win32::Shortcut;
-	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $raw_subfile));
-	    if ($shortcut) {
-		# The file to be processed is now the target of the shortcut
-		$this_file_base_dir = "";
-		$file = "";
-		$raw_subfile = $shortcut->Path;
-	    }
-	}
-
-	# check for a symlink pointing back to a leading directory
-	if (-d "$dirname/$raw_subfile" && -l "$dirname/$raw_subfile") {
-	    # readlink gives a "fatal error" on systems that don't implement
-	    # symlinks. This assumes the the -l test above would fail on those.
-	    my $linkdest=readlink "$dirname/$raw_subfile";
-	    if (!defined ($linkdest)) {
-		# system error - file not found?
-		warn "DirectoryPlugin: symlink problem - $!";
-	    } else {
-		# see if link points to current or a parent directory
-		if ($linkdest =~ m@^[\./\\]+$@ ||
-		    index($dirname, $linkdest) != -1) {
-		    warn "DirectoryPlugin: Ignoring recursive symlink ($dirname/$raw_subfile -> $linkdest)\n";
-		    next;
-		    ;
-		}
-	    }
-	}
-
-	print $outhandle "DirectoryPlugin: preparing metadata for $raw_subfile\n" if ($verbosity > 2);
-
-	# Make a copy of $in_metadata to pass to $raw_subfile
-	my $out_metadata = {};
-	&metadatautil::combine_metadata_structures($out_metadata, $in_metadata);
-
-	# check the assocfile_info
-	if (defined $self->{'assocfile_info'}->{$raw_full_filename}) {
-	    &metadatautil::combine_metadata_structures($out_metadata, $self->{'assocfile_info'}->{$raw_full_filename});
-	}
-
-	# $subfile by this point is url-encoded => all ASCII chars => no need to encode as UTF8
-
-	# Next add metadata read in XML files (if it is supplied)
-	if ($additionalmetadata == 1) {
-	    foreach my $filespec (@extrametakeys) {
-		## use the url-encoded filename to do the filename comparison
-
-		if ($subfile =~ /^$filespec$/) {
-		    print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
-			if ($verbosity > 2);
-		    my $mdref = $extrametadata{$filespec};
-		    my $mfref = $extrametafile{$filespec};
-
-		    # Add the list files where the metadata came from
-		    # into the metadata table so we can track this
-		    # This mechanism is similar to how gsdlassocfile works
-
-		    my @metafile_pair = ();
-		    foreach my $l (keys %$mfref) {
-			my $f = $mfref->{$l};
-			push (@metafile_pair, "$f : $l");
-		    }
-
-		    $mdref->{'gsdlmetafile'} = \@metafile_pair;
-
-		    &metadatautil::combine_metadata_structures($out_metadata, $mdref);
-		}
-	    }
-	}
-
-        # So, here's my problem. I have an empty archives directory, and a 
-        # manifest file that says to add 1 out of 2 documents in the import
-        # directory. The directory scan finds both files, and doc 1 is in
-        # new_files, so that is all good. But doc 2 is also processed, and
-        # without any blocking or other rule, also gets imported.
-        # If we are processing a Manifest then files (not docs) that aren't in
-        # either the new files or reindex files are skipped.
-        # We'll also test for metadata files here, even though they should
-        # have been handled above - just in case we have the weird case where
-        # a file is both a metadata source and a file in its own right. [jmt12]
-        if (defined $block_hash
-            && defined $block_hash->{'manifest'}
-            && $block_hash->{'manifest'} eq 'true'
-            && !-d $raw_full_filename
-            && !$block_hash->{'new_files'}->{$raw_full_filename}
-            && !$block_hash->{'reindex_files'}->{$raw_full_filename}
-            && !$block_hash->{'metadata_files'}->{$raw_full_filename})
-        {
-          print $outhandle "**** Not in manifest. Skipping $subfile\n" if ($verbosity > 3);
-          next;
-        }
-
-	if (defined $self->{'inf_timestamp'}) {
-	    # Look to see if it's a completely new file
-
-	    if (!$block_hash->{'new_files'}->{$raw_full_filename}) {
-		# Not a new file, must be an existing file
-		# Let' see if it's newer than the last import.pl
-
-
-		if (! -d $raw_full_filename) {
-		    if (!$block_hash->{'reindex_files'}->{$raw_full_filename}) {
-			# filename has been around for longer than inf_timestamp
-			print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
-			next;
-		    }
-		    else {
-			# Remove old folder in archives (might hash to something different)
-			# *** should be doing this on a Del one as well
-			# but leave folder name?? and ensure hashs to
-			# same again??
-
-			# Then let through as new doc??
-
-			# mark to doc-oids that rely on it for re-indexing
-		    }
-		}
-	    }
-        }
-
-	# Recursively read each $subfile
-	print $outhandle "DirectoryPlugin recurring: $subfile\n" if ($verbosity > 2);
-	
-	$count += &plugin::read ($pluginfo, $this_file_base_dir,
-				 $raw_file_subfile, $block_hash,
-				 $out_metadata, $processor, $maxdocs, ($total_count + $count), $gli);
-    }
-
-    return $count;
-}
-
-sub compile_stats {
-    my $self = shift(@_);
-    my ($stats) = @_;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/plugins/PagedImagePlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/PagedImagePlugin.pm	(revision 26944)
+++ 	(revision )
@@ -1,761 +1,0 @@
-###########################################################################
-#
-# PagedImagePlugin.pm -- plugin for sets of images and OCR text that
-#  make up a document
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# PagedImagePlugin
-# processes sequences of images, with optional OCR text
-#
-# This plugin takes *.item files, which contain metadata and lists of image 
-# files, and produces a document containing sections, one for each page.
-# The files should be named something.item, then you can have more than one 
-# book in a directory. You will need to create these files, one for each
-# document/book.
-#
-#There are two formats for the item files: a plain text format, and an xml 
-#format. You can use either format, and can have both formats in the same 
-#collection if you like. If you use the plain format, you must not start the 
-#file off with <PagedDocument>
-
-#### PLAIN FORMAT
-# The format of the xxx.item file is as follows:
-# The first lines contain any metadata for the whole document
-# <metadata-name>metadata-value
-# eg.
-# <Title>Snail farming
-# <Date>19230102
-# Then comes a list of pages, one page per line, each line has the format
-#
-# pagenum:imagefile:textfile:r
-#
-# page num and imagefile are required. pagenum is used for the Title 
-# of the section, and in the display is shown as page <pagenum>. 
-# imagefile is the image for the page. textfile is an optional text 
-# file containing the OCR (or any) text for the page - this gets added 
-# as the text for the section. r is optional, and signals that the image 
-# should be rotated 180deg. Eg use this if the image has been made upside down.
-# So an example item file looks like:
-# <Title>Snail farming
-# <Date>19960403
-# 1:p1.gif:p1.txt:
-# 2:p2.gif::
-# 3:p3.gif:p3.txt:
-# 3b:p3b.gif:p3b.txt:r
-# The second page has no text, the fourth page is a back page, and 
-# should be rotated.
-# 
-
-#### XML FORMAT
-# The xml format looks like the following
-#<PagedDocument>
-#<Metadata name="Title">The Title of the entire document</Metadata>
-#<Page pagenum="1" imgfile="xxx.jpg" txtfile="yyy.txt">
-#<Metadata name="Title">The Title of this page</Metadata>
-#</Page>
-#... more pages
-#</PagedDocument>
-#PagedDocument contains a list of Pages, Metadata and PageGroups. Any metadata 
-#that is not inside another tag will belong to the document.
-#Each Page has a pagenum (not used at the moment), an imgfile and/or a txtfile.
-#These are both optional - if neither is used, the section will have no content.
-#Pages can also have metadata associated with them.
-#PageGroups can be introduced at any point - they can contain Metadata and Pages and other PageGroups. They are used to introduce hierarchical structure into the document.
-#For example
-#<PagedDocument>
-#<PageGroup>
-#<Page>
-#<Page>
-#</PageGroup>
-#<Page>
-#</PagedDocument>
-#would generate a structure like
-#X
-#--X
-#  --X
-#  --X
-#--X
-#PageGroup tags can also have imgfile/textfile metadata if you like - this way they get some content themselves.
-
-#Currently the XML structure doesn't work very well with the paged document type, unless you use numerical Titles for each section.
-#There is still a bit of work to do on this format:
-#* enable other text file types, eg html, pdf etc
-#* make the document paging work properly
-#* add pagenum as Title unless a Title is present?
-
-# All the supplemetary image amd text files should be in the same folder as 
-# the .item file.
-#
-# To display the images instead of the document text, you can use [srcicon] 
-# in the DocumentText format statement.
-# For example, 
-#
-# format DocumentText "<center><table width=_pagewidth_><tr><td>[srcicon]</td></tr></table></center>"
-#
-# To have it create thumbnail size images, use the '-create_thumbnail' option.
-# To have it create medium size images for display, use the '-create_screenview'
-# option. As usual, running 
-# 'perl -S pluginfo.pl PagedImagePlugin' will list all the options.
-
-# If you want the resulting documents to be presented with a table of 
-# contents, use '-documenttype hierarchy', otherwise they will have 
-# next and previous arrows, and a goto page X box. 
-
-# If you have used -create_screenview, you can also use [screenicon] in the format
-# statement to display the smaller image.  Here is an example  that switches 
-# between the two: 
-#
-# format DocumentText "<center><table width=_pagewidth_><tr><td>{If}{_cgiargp_ eq full,<a href='_httpdocument_&d=_cgiargd_&p=small'>Switch to small version.</a>,<a href='_httpdocument_&d=_cgiargd_&p=full'>Switch to fullsize version</a>}</td></tr><tr><td>{If}{_cgiargp_ eq full,<a href='_httpdocument_&d=_cgiargd_&p=small' title='Switch to small version'>[srcicon]</a>,<a href='_httpdocument_&d=_cgiargd_&p=full' title='Switch to fullsize version'>[screenicon]</a>}</td></tr></table></center>"
-#
-# Additional metadata can be added into the .item files, alternatively you can 
-# use normal metadata.xml files, with the name of the xxx.item file as the 
-# FileName (only for document level metadata).
-
-package PagedImagePlugin;
-
-use Encode;
-use ReadXMLFile;
-use ReadTextFile;
-use ImageConverter;
-use MetadataRead;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-sub BEGIN {
-    @PagedImagePlugin::ISA = ('MetadataRead', 'ReadXMLFile', 'ReadTextFile', 'ImageConverter');
-}
-
-my $type_list =
-    [ { 'name' => "paged",
-        'desc' => "{PagedImagePlugin.documenttype.paged}" },
-      { 'name' => "hierarchy",
-        'desc' => "{PagedImagePlugin.documenttype.hierarchy}" } ];
-
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BasePlugin.process_exp}",
-	'type' => "string",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "title_sub",
-	'desc' => "{HTMLPlugin.title_sub}",
-	'type' => "string", 
-	'deft' => "" },
-      { 'name' => "headerpage",
-	'desc' => "{PagedImagePlugin.headerpage}",
-	'type' => "flag",
-	'reqd' => "no" },
-      { 'name' => "documenttype",
-	'desc' => "{PagedImagePlugin.documenttype}",
-	'type' => "enum",
-	'list' => $type_list,
-	'deft' => "paged",
-	'reqd' => "no" },
-      {'name' => "processing_tmp_files",
-       'desc' => "{BasePlugin.processing_tmp_files}",
-       'type' => "flag",
-       'hiddengli' => "yes"}
-];
-
-
-my $options = { 'name'     => "PagedImagePlugin",
-		'desc'     => "{PagedImagePlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-    
-    my $imc_self = new ImageConverter($pluginlist, $inputargs, $hashArgOptLists);
-    my $rtf_self = new ReadTextFile($pluginlist, $inputargs, $hashArgOptLists, 1);
-    my $rxf_self = new ReadXMLFile($pluginlist, $inputargs, $hashArgOptLists);
-
-    my $self = BasePlugin::merge_inheritance($imc_self,$rtf_self,$rxf_self);
-
-    # Update $self used by XML::Parser so it finds callback functions 
-    # such as start_document here and not in ReadXMLFile (which is what
-    # $self was when new XML::Parser was done)
-    #
-    # If the $self returned by this constructor is the same as the one
-    # used in ReadXMLFile (e.g. in the GreenstoneXMLPlugin) then this step isn't necessary
-    #
-    # Consider embedding this type of assignment into merge_inheritance
-    # to help catch all cases?
-
-    $rxf_self->{'parser'}->{'PluginObj'} = $self;
-
-    return bless $self, $class;
-}
-
-
-sub init {
-    my $self = shift (@_);
-    my ($verbosity, $outhandle, $failhandle) = @_;
-
-    $self->SUPER::init(@_);
-    $self->ImageConverter::init();
-}
-
-sub begin {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    $self->SUPER::begin(@_);
-    $self->ImageConverter::begin(@_);
-}
-
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^\.item$^;
-}
-
-sub get_doctype {
-    my $self = shift(@_);
-    
-    return "PagedDocument";
-}
-
-
-# want to use BasePlugin's version of this, not ReadXMLFile's
-sub can_process_this_file {
-    my $self = shift(@_);
-    return $self->BasePlugin::can_process_this_file(@_);
-}
-
-# instead of a block exp, now we scan the file and record all text and img files mentioned there for blocking.
-sub store_block_files
-{
-    my $self = shift (@_);
-    my ($filename_full_path, $block_hash) = @_;
-
-    my $xml_version = $self->is_xml_item_file($filename_full_path);
-    
-    # do we need to do this? 
-    # does BOM interfere just with XML parsing? In that case don't need it here
-    # if we do it here, we are modifying the file before we have worked out if
-    # its new or not, so it will always be reimported.
-    #$self->tidy_item_file($filename_full_path);
-
-    my ($dir, $file) = $filename_full_path =~ /^(.*?)([^\/\\]*)$/;
-    if ($xml_version) {
-
-	# do something
-	$self->scan_xml_for_files_to_block($filename_full_path, $dir, $block_hash);
-    } else {
-	
-	$self->scan_item_for_files_to_block($filename_full_path, $dir, $block_hash);
-    }
-	
-}
-
-# we want to use BasePlugin's read, not ReadXMLFile's
-sub read
-{
-    my $self = shift (@_);
-
-    $self->BasePlugin::read(@_);
-}
-
-
-
-sub read_into_doc_obj {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-    my $outhandle = $self->{'outhandle'};
-    my $verbosity = $self->{'verbosity'};
-    
-    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-
-    print $outhandle "PagedImagePlugin processing \"$filename_full_path\"\n"
-	if $verbosity > 1;
-    print STDERR "<Processing n='$file' p='PagedImagePlugin'>\n" if ($gli);
-    
-    $self->{'MaxImageWidth'} = 0;
-    $self->{'MaxImageHeight'} = 0;
-    
-    # here we need to decide if we have an old text .item file, or a new xml 
-    # .item file
-    my $xml_version = $self->is_xml_item_file($filename_full_path);
-
-    $self->tidy_item_file($filename_full_path);
-    
-    my $doc_obj;
-    if ($xml_version) {
-	# careful checking needed here!! are we using local xml handlers or super ones
-	$self->ReadXMLFile::read($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli);
-	$doc_obj = $self->{'doc_obj'};
-    } else {
-	my ($dir);
-	($dir, $file) = $filename_full_path =~ /^(.*?)([^\/\\]*)$/;
-
-	#process the .item file
-	$doc_obj = $self->process_item($filename_full_path, $dir, $file, $processor, $metadata);
-	
-    }
-
-    my $section = $doc_obj->get_top_section();
-        
-    $doc_obj->add_utf8_metadata($section, "Plugin", "$self->{'plugin_type'}");
-    $doc_obj->add_metadata($section, "FileFormat", "PagedImage");
-
-    # include any metadata passed in from previous plugins 
-    # note that this metadata is associated with the top level section
-    $self->add_associated_files($doc_obj, $filename_full_path);
-    $self->extra_metadata ($doc_obj, $section, $metadata);
-    $self->auto_extract_metadata ($doc_obj);
-
-    # if we haven't found any Title so far, assign one
-    $self->title_fallback($doc_obj,$section,$filename_no_path);
-
-    $self->add_OID($doc_obj);
-    return (1,$doc_obj);
-}
-
-# for now, the test is if the first non-empty line is <PagedDocument>, then its xml
-sub is_xml_item_file {
-    my $self = shift(@_);
-    my ($filename) = @_;
-
-    my $xml_version = 0;
-    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
-
-    my $line = "";
-    my $num = 0;
-
-    $line = <ITEMFILE>;
-    while (defined ($line) && ($line !~ /\w/)) {
-	$line = <ITEMFILE>;
-    }
-
-    if (defined $line) {
-	chomp $line;
-	if ($line =~ /<PagedDocument/) {
-	    $xml_version = 1;
-	}
-    } 
-
-    close ITEMFILE; 
-    return $xml_version;
-}
-
-sub tidy_item_file {
-    my $self = shift(@_);
-    my ($filename) = @_;
-
-    open (ITEMFILE, $filename) || die "couldn't open $filename\n";
-    my $backup_filename = "backup.item";
-    open (BACKUP,">$backup_filename")|| die "couldn't write to $backup_filename\n";
-    my $line = "";
-    $line = <ITEMFILE>;
-    $line =~ s/^\xEF\xBB\xBF//; # strip BOM
-    $line =~ s/\x0B+//ig;
-    $line =~ s/&/&amp;/g;
-    print BACKUP ($line);
-    #Tidy up the item file some metadata title contains \vt-vertical tab
-    while ($line = <ITEMFILE>) {
-	$line =~ s/\x0B+//ig;
-	$line =~ s/&/&amp;/g;
-	print BACKUP ($line);
-    }
-    close ITEMFILE;
-    close BACKUP;
-    &File::Copy::copy ($backup_filename, $filename);
-    &util::rm($backup_filename);
-
-}
-
-sub rotate_image {
-    my $self = shift (@_);
-    my ($filename_full_path) = @_;
-    
-    my ($this_filetype) = $filename_full_path =~ /\.([^\.]*)$/;
-    my $result = $self->convert($filename_full_path, $this_filetype, "-rotate 180", "ROTATE");
-    my ($new_filename) = ($result =~ /=>(.*\.$this_filetype)/);
-    if (-e "$new_filename") {
-	return $new_filename;
-    }
-    # somethings gone wrong
-    return $filename_full_path;
-
-}
-
-sub process_image {
-    my $self = shift(@_);
-    my ($filename_full_path, $filename_no_path, $doc_obj, $section, $rotate) = @_;
-    # check the filenames
-    return 0 if ($filename_no_path eq "" || !-f $filename_full_path);
- 
-    # remember that this image file was one of our source files, but only 
-    # if we are not processing a tmp file
-    if (!$self->{'processing_tmp_files'} ) {
-	$doc_obj->associate_source_file($filename_full_path);
-    }
-    # do rotation
-    if  ((defined $rotate) && ($rotate eq "r")) {
-	# we get a new temporary file which is rotated
-	$filename_full_path = $self->rotate_image($filename_full_path);
-    }
-    
-    # do generate images
-    my $result = 0;
-    if ($self->{'image_conversion_available'} == 1) {
-	# do we need to convert $filename_no_path to utf8/url encoded? 
-	# We are already reading in from a file, what encoding is it in???
-	my $url_encoded_full_filename 
-	    = &unicode::raw_filename_to_url_encoded($filename_full_path);
-	$result = $self->generate_images($filename_full_path, $url_encoded_full_filename, $doc_obj, $section);
-    }
-    #overwrite one set in ImageConverter
-    $doc_obj->set_metadata_element ($section, "FileFormat", "PagedImage");
-    return $result;
-}
-
-
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-    $self->{'element'} = $element;
-    
-    my $doc_obj = $self->{'doc_obj'};
-    if ($element eq "PagedDocument") {
-	$self->{'current_section'} = $doc_obj->get_top_section();
-    } elsif ($element eq "PageGroup" || $element eq "Page") {
-	# create a new section as a child
-	$self->{'current_section'} = $doc_obj->insert_section($doc_obj->get_end_child($self->{'current_section'}));
-	$self->{'num_pages'}++;
-	# assign pagenum as  what??
-	my $pagenum = $_{'pagenum'}; #TODO!!
-	if (defined $pagenum) {
-	    $doc_obj->set_utf8_metadata_element($self->{'current_section'}, 'PageNum', $pagenum);
-	}
-	my ($imgfile) = $_{'imgfile'};
-	if (defined $imgfile) {
-	    # *****
-	    # What about support for rotate image (e.g. old ':r' notation)? 
-	    $self->process_image($self->{'xml_file_dir'}.$imgfile, $imgfile, $doc_obj, $self->{'current_section'});
-	}
-	my ($txtfile) = $_{'txtfile'};
-	if (defined($txtfile)&& $txtfile ne "") {
-	    $self->process_text ($self->{'xml_file_dir'}.$txtfile, $txtfile, $doc_obj, $self->{'current_section'});
-  	} else {
-	    $self->add_dummy_text($doc_obj, $self->{'current_section'});
-	}
-    } elsif ($element eq "Metadata") {
-	$self->{'metadata_name'} = $_{'name'};
-    }
-}
-
-sub xml_end_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-    
-    my $doc_obj = $self->{'doc_obj'};
-    if ($element eq "Page" || $element eq "PageGroup") {
-	# if Title hasn't been assigned, set PageNum as Title
-	if (!defined $doc_obj->get_metadata_element ($self->{'current_section'}, "Title") && defined $doc_obj->get_metadata_element ($self->{'current_section'}, "PageNum" )) {
-	    $doc_obj->add_utf8_metadata ($self->{'current_section'}, "Title", $doc_obj->get_metadata_element ($self->{'current_section'}, "PageNum" ));
-	}
-	# move the current section back to the parent
-	$self->{'current_section'} = $doc_obj->get_parent_section($self->{'current_section'});
-    } elsif ($element eq "Metadata") {
-	
-	# text read in by XML::Parser is in Perl's binary byte value
-	# form ... need to explicitly make it UTF-8		
-	my $meta_name = decode("utf-8",$self->{'metadata_name'});
-	my $metadata_value = decode("utf-8",$self->{'metadata_value'});
-	
-	if ($meta_name =~ /\./) {
-	    $meta_name = "ex.$meta_name";
-	}
-	
-	$doc_obj->add_utf8_metadata ($self->{'current_section'}, $meta_name, $metadata_value);
-	$self->{'metadata_name'} = "";
-	$self->{'metadata_value'} = "";
-
-    }
-    # otherwise we ignore the end tag
-}
-
-
-sub xml_text {
-    my $self = shift(@_);
-    my ($expat) = @_; 
-
-    if ($self->{'element'} eq "Metadata" && $self->{'metadata_name'}) {
-   	$self->{'metadata_value'} .= $_;
-    }
-}
-
-sub xml_doctype {
-}
-
-sub open_document {
-    my $self = shift(@_);
-    
-    # create a new document
-    $self->{'doc_obj'} = new doc ($self->{'filename'}, "indexed_doc", $self->{'file_rename_method'});
-    # TODO is file filenmae_no_path??
-    $self->set_initial_doc_fields($self->{'doc_obj'}, $self->{'filename'}, $self->{'processor'}, $self->{'metadata'});
-
-    my ($dir, $file) = $self->{'filename'} =~ /^(.*?)([^\/\\]*)$/;
-    $self->{'xml_file_dir'} = $dir;
-    $self->{'num_pages'} = 0;
-
-}
-
-sub close_document {
-    my $self = shift(@_);
-    my $doc_obj = $self->{'doc_obj'};
-    
-    # add numpages metadata
-    my $topsection = $doc_obj->get_top_section();
-
-    $doc_obj->set_utf8_metadata_element ($topsection, 'NumPages', $self->{'num_pages'});
-
-    $doc_obj->set_utf8_metadata_element($topsection,"MaxImageWidth",$self->{'MaxImageWidth'});
-    $doc_obj->set_utf8_metadata_element($topsection,"MaxImageHeight",$self->{'MaxImageHeight'});
-    $self->{'MaxImageWidth'} = undef;
-    $self->{'MaxImageHeight'} = undef;
-    
-}
-
-
-sub set_initial_doc_fields {
-    my $self = shift(@_);
-    my ($doc_obj, $filename_full_path, $processor, $metadata) = @_;
-
-    my $topsection = $doc_obj->get_top_section();
-
-    if ($self->{'documenttype'} eq 'paged') {
-	# set the gsdlthistype metadata to Paged - this ensures this document will
-	# be treated as a Paged doc, even if Titles are not numeric
-	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Paged");
-    } else {
-	$doc_obj->set_utf8_metadata_element ($topsection, "gsdlthistype", "Hierarchy");
-    }
-
-    my $plugin_filename_encoding = $self->{'filename_encoding'};
-    my $filename_encoding = $self->deduce_filename_encoding($filename_full_path,$metadata,$plugin_filename_encoding);
-    $self->set_Source_metadata($doc_obj, $filename_full_path, $filename_encoding);
-   
-    # if we want a header page, we need to add some text into the top section, otherwise this section will become invisible
-    if ($self->{'headerpage'}) {
-	$self->add_dummy_text($doc_obj, $topsection);
-    }
-}
-
-sub scan_xml_for_files_to_block
-{
-    my $self = shift (@_);
-    my ($filename_full_path, $dir, $block_hash) = @_;
-
-    open (ITEMFILE, $filename_full_path) || die "couldn't open $filename_full_path to work out which files to block\n";
-    my $line = "";
-    while (defined ($line = <ITEMFILE>)) {
-	next unless $line =~ /\w/;
-
-	if ($line =~ /imgfile=\"([^\"]+)\"/) {
-	    &util::block_filename($block_hash,&util::filename_cat($dir,$1));
-	}
-	if ($line =~ /txtfile=\"([^\"]+)\"/) {
-	    &util::block_filename($block_hash,&util::filename_cat($dir,$1));
-	}
-    }
-    close ITEMFILE;
-    
-}
-
-sub scan_item_for_files_to_block
-{
-    my $self = shift (@_);
-    my ($filename_full_path, $dir, $block_hash) = @_;
-
-    open (ITEMFILE, $filename_full_path) || die "couldn't open $filename_full_path to work out which files to block\n";
-    my $line = "";
-    while (defined ($line = <ITEMFILE>)) {
-	next unless $line =~ /\w/;
-	chomp $line;
-	next if $line =~ /^#/; # ignore comment lines
-	next if ($line =~ /^<([^>]*)>\s*(.*?)\s*$/); # ignore metadata lines 
-	# line should be like page:imagefilename:textfilename:r 
-	$line =~ s/^\s+//; #remove space at the front
-	$line =~ s/\s+$//; #remove space at the end
-	my ($pagenum, $imgname, $txtname, $rotate) = split /:/, $line;
-	    
-	# find the image file if there is one
-	if (defined $imgname && $imgname ne "") {
-	    &util::block_filename($block_hash, &util::filename_cat( $dir,$imgname));
-	}
-	# find the text file if there is one
-	if (defined $txtname && $txtname ne "") {
-	    &util::block_filename($block_hash, &util::filename_cat($dir,$txtname));
-	}
-    }
-    close ITEMFILE;
-
-}
-
-sub process_item {
-    my $self = shift (@_);
-    my ($filename_full_path, $dir, $filename_no_path, $processor, $metadata) = @_;
-
-    my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
-    $self->set_initial_doc_fields($doc_obj, $filename_full_path, $processor, $metadata);
-    my $topsection = $doc_obj->get_top_section();
-    open (ITEMFILE, $filename_full_path) || die "couldn't open $filename_full_path\n";
-    my $line = "";
-    my $num = 0;
-    while (defined ($line = <ITEMFILE>)) {
-	
-	# Since process_item is called not on an XML item file, but a text item file
-	# don't decode into UTF8 the text that was read in, since it's already UTF-8	
-	#$line = decode("utf-8",$line);
-	
-	next unless $line =~ /\w/;
-	chomp $line;
-	next if $line =~ /^#/; # ignore comment lines
-	if ($line =~ /^<([^>]*)>\s*(.*?)\s*$/) {
-	    my $meta_name = $1;
-	    my $meta_value = $2;
-	    if ($meta_name =~ /\./) {
-		$meta_name = "ex.$meta_name";
-	    }
-	    $doc_obj->set_utf8_metadata_element ($topsection, $meta_name, $meta_value);
-	    #$meta->{$1} = $2;
-	} else {
-	    $num++;
-	    # line should be like page:imagefilename:textfilename:r - the r is optional -> means rotate the image 180 deg
-	    $line =~ s/^\s+//; #remove space at the front
-	    $line =~ s/\s+$//; #remove space at the end
-	    my ($pagenum, $imgname, $txtname, $rotate) = split /:/, $line;
-	    
-	    # create a new section for each image file
-	    my $cursection = $doc_obj->insert_section($doc_obj->get_end_child($topsection));
-	    # the page number becomes the Title
-	    $doc_obj->set_utf8_metadata_element($cursection, 'Title', $pagenum);
-	 
-	    # process the image for this page if there is one
-	    if (defined $imgname && $imgname ne "") {
-		my $result1 = $self->process_image($dir.$imgname, $imgname, $doc_obj, $cursection, $rotate);
-		if (!defined $result1)
-		{
-		    print "PagedImagePlugin: couldn't process image \"$dir$imgname\" for item \"$filename_full_path\"\n";
-		}
-	    }
-	    # process the text file if one is there
-	    if (defined $txtname && $txtname ne "") {
-		my $result2 = $self->process_text ($dir.$txtname, $txtname, $doc_obj, $cursection);
-               
-		if (!defined $result2) {
-		    print "PagedImagePlugin: couldn't process text file \"$dir.$txtname\" for item \"$filename_full_path\"\n";
-		    $self->add_dummy_text($doc_obj, $cursection);
-		}
-	    } else {
-		# otherwise add in some dummy text 
-		$self->add_dummy_text($doc_obj, $cursection);
-	    }
-	}
-    }
-    
-    close ITEMFILE;
-
-    # add numpages metadata
-    $doc_obj->set_utf8_metadata_element ($topsection, 'NumPages', "$num");
-
-    $doc_obj->set_utf8_metadata_element($topsection,"MaxImageWidth",$self->{'MaxImageWidth'});
-    $doc_obj->set_utf8_metadata_element($topsection,"MaxImageHeight",$self->{'MaxImageHeight'});
-    $self->{'MaxImageWidth'} = undef;
-    $self->{'MaxImageHeight'} = undef;
-
-
-    return $doc_obj;
-}
-
-sub process_text {
-    my $self = shift (@_);
-    my ($filename_full_path, $file, $doc_obj, $cursection) = @_;
-    
-    # check that the text file exists!!
-    if (!-f $filename_full_path) {
-	print "PagedImagePlugin: ERROR: File $filename_full_path does not exist, skipping\n";
-	return 0;
-    }
-
-    # remember that this text file was one of our source files, but only 
-    # if we are not processing a tmp file
-    if (!$self->{'processing_tmp_files'} ) {
-	$doc_obj->associate_source_file($filename_full_path);
-    }
-    # Do encoding stuff
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
-
-    my $text="";
-    &ReadTextFile::read_file($self, $filename_full_path, $encoding, $language, \$text); # already decoded as utf8
-    if (!length ($text)) {
-	# It's a bit unusual but not out of the question to have no text, so just give a warning
-        print "PagedImagePlugin: WARNING: $filename_full_path contains no text\n";
-    }
-
-    # we need to escape the escape character, or else mg will convert into
-    # eg literal newlines, instead of leaving the text as '\n'
-    $text =~ s/\\/\\\\/g; # macro language
-    $text =~ s/_/\\_/g; # macro language
-
-
-    if ($text =~ m/<html.*?>\s*<head.*?>.*<\/head>\s*<body.*?>(.*)<\/body>\s*<\/html>\s*$/is) {
-	# looks like HTML input
-	# no need to escape < and > or put in <pre> tags
-
-	$text = $1;
-
-	# add text to document object
-	$doc_obj->add_utf8_text($cursection, "$text");
-    }
-    else {
-	$text =~ s/</&lt;/g;
-	$text =~ s/>/&gt;/g;
-
-	# insert preformat tags and add text to document object
-	$doc_obj->add_utf8_text($cursection, "<pre>\n$text\n</pre>");
-    }
-
-    
-    return 1;
-}
-
-
-sub clean_up_after_doc_obj_processing {
-    my $self = shift(@_);
-    
-    $self->ImageConverter::clean_up_temporary_files();
-}
-
-1;
