Index: /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm	(revision 30288)
+++ /gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm	(revision 30289)
@@ -34,4 +34,5 @@
 use plugin;
 use util;
+use FileUtils;
 use metadatautil;
 
@@ -41,5 +42,7 @@
 no strict 'subs';
 
+use Encode::Locale;
 use Encode;
+use Unicode::Normalize;
 
 BEGIN {
@@ -79,5 +82,7 @@
 
     my $self = new PrintInfo($pluginlist, $inputargs, $hashArgOptLists);
-    
+
+    print STDERR "INFO: This DirectoryPlugin supports version 2 manifest files\n";
+
     if ($self->{'info_only'}) {
 	# don't worry about any options or initialisations etc
@@ -89,5 +94,5 @@
 	die "ERROR: DirectoryPlugin -use_metadata_files option has been deprecated. Please remove the option and add MetadataXMLPlug to your plugin list instead!\n";
     }
-	
+
     $self->{'num_processed'} = 0;
     $self->{'num_not_processed'} = 0;
@@ -134,6 +139,6 @@
     	my $archives_inf = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $output_dir);
 
-	if ( &FileUtils::fileExists($archives_inf) ) {
-	    $self->{'inf_timestamp'} = &FileUtils::file_lastmodified($archives_inf);
+	if ( -e $archives_inf ) {
+	    $self->{'inf_timestamp'} = -M $archives_inf;
 	}
     }
@@ -172,5 +177,5 @@
     my $self = shift (@_);
     
-    return '(?i)(CVS|\.svn|Thumbs\.db|OIDcount|~)$';
+    return '(?i)(CVS|\.svn|Thumbs\.db|OIDcount|\.DS_Store|~)$';
 }
 
@@ -179,7 +184,6 @@
     my $self = shift(@_);
     my ($dirname) = @_;
-
-    # replace -d with function in util library
-    return undef unless (&FileUtils::directoryExists($dirname));
+    
+    return undef unless (-d $dirname);
 
     return 0 if ($self->{'block_exp'} ne "" && $dirname =~ /$self->{'block_exp'}/);
@@ -259,7 +263,6 @@
 
     $filename_full_path = &util::upgrade_if_dos_filename($filename_full_path);
-###    print STDERR "*** DirectoryPlugin::file_is_blocked $filename_full_path\n";
-
-    if ($ENV{'GSDLOS'} =~ m/^windows$/) {
+
+    if (($ENV{'GSDLOS'} =~ m/^windows$/) && ($^O ne "cygwin")) {
 	# on windows, all block paths are lowercased.
 	my $lower_filename = lc ($filename_full_path);
@@ -295,10 +298,10 @@
     # that it is not explicitly blocked.
     my $dirname = $file;
-    $dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    $dirname = &FileUtils::filenameConcatenate($base_dir, $file) if $base_dir =~ /\w/;
 
     my $directory_ok = $self->check_directory_path($dirname);
     return $directory_ok unless (defined $directory_ok && $directory_ok == 1);
 
-    print $outhandle "File scan checking directory: $dirname\n";
+    print $outhandle "Global file scan checking directory: $dirname\n";
 
     $block_hash->{'all_files'} = {} unless defined $block_hash->{'all_files'};
@@ -308,19 +311,21 @@
     $block_hash->{'shared_fileroot'} = {} unless defined $block_hash->{'shared_fileroot'};
 
-    # Recur over directory contents.
+     # Recur over directory contents.
+    my (@dir, $subfile);
+    #my $count = 0;
+    
     print $outhandle "DirectoryPlugin block: getting directory $dirname\n" if ($verbosity > 2);
-
+    
     # find all the files in the directory
-    my @dir = @{&FileUtils::readDirectory($dirname)};
-    if (scalar(@dir) == 0)
-    {
-      if ($gli)
-      {
-        print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
-      }
-      print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
-      return -1; # error in processing
-    }
-
+    if (!opendir (DIR, $dirname)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
+	}
+	print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
+	return -1; # error in processing
+    }
+    @dir = sort readdir (DIR);
+    closedir (DIR);
+    
     for (my $i = 0; $i < scalar(@dir); $i++) {
 	my $raw_subfile = $dir[$i];
@@ -328,8 +333,9 @@
 
 	my $this_file_base_dir = $base_dir;
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
 
 	# Recursively read each $raw_subfile
-	print $outhandle "DirectoryPlugin block recurring: $raw_file_subfile\n" if ($verbosity > 2);
+	print $outhandle "DirectoryPlugin block recurring: ". Encode::decode("utf8", $raw_file_subfile) ."\n" if ($verbosity > 2);
+	print $outhandle "DirectoryPlugin block recurring: ". Encode::decode(locale =>$raw_file_subfile) ."\n" if ($verbosity > 2);
 	
 	#$count += &plugin::file_block_read ($pluginfo, $this_file_base_dir,
@@ -369,5 +375,4 @@
     my $self = shift (@_);
     my ($pluginfo, $base_dir, $file, $block_hash, $in_metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
     my $outhandle = $self->{'outhandle'};
     my $verbosity = $self->{'verbosity'};
@@ -380,5 +385,5 @@
     } else {
 	$dirname = $file;
-	$dirname = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+	$dirname = &FileUtils::filenameConcatenate($base_dir, $file) if $base_dir =~ /\w/;
     }
 	
@@ -393,27 +398,44 @@
 
     # Recur over directory contents.
+    my @dir;
+    
     print $outhandle "DirectoryPlugin read: getting directory $dirname\n" if ($verbosity > 2);
-
+    
     # find all the files in the directory
-    my @dir = @{&FileUtils::readDirectory($dirname)};
-    if (scalar(@dir) == 0)
-    {
-      if ($gli)
-      {
-        print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
-      }
-      print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
-      return -1; # error in processing
-    }
-    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dir;
-
-    # Re-order the files in the list so any directories ending with .all are
-    # moved to the end
+    if (!opendir (DIR, $dirname)) {
+	if ($gli) {
+	    print STDERR "<ProcessingError n='$file' r='Could not read directory $dirname'>\n";
+	}
+	print $outhandle "DirectoryPlugin: WARNING - couldn't read directory $dirname\n";
+	return -1; # error in processing
+    }
+    @dir = sort readdir (DIR);
+    map {  $_ = &unicode::raw_filename_to_url_encoded($_);  } @dir;
+    closedir (DIR);
+    # Re-order the files in the list so any directories ending with .all are moved to the end
     for (my $i = scalar(@dir) - 1; $i >= 0; $i--) {
-	if (-d &util::filename_cat($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
+	if (-d &FileUtils::filenameConcatenate($dirname, $dir[$i]) && $dir[$i] =~ /\.all$/) {
 	    push(@dir, splice(@dir, $i, 1));
 	}
     }
 
+    # Chain through to the rest of the read function (now split off and named
+    # read_phase2)
+    my $count = $self->read_phase2($pluginfo, $dirname, \@dir, $base_dir, $file, $block_hash, $in_metadata, $processor, $maxdocs, $total_count, $gli);
+
+    return $count;
+}
+
+sub read_phase2
+{
+    my $self = shift (@_);
+    my ($pluginfo, $dirname, $dir_ref, $base_dir, $file, $block_hash, $in_metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+    # These were defined in read (phase 1)
+    my @dir = @{$dir_ref};
+    my $subfile;
+
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+    
     # setup the metadata structures. we do a metadata_read pass to see if there is any additional metadata, then pass it to read
     
@@ -428,7 +450,7 @@
     my $base_dir_regexp = $base_dir;
     $base_dir_regexp =~ s/\//$os_dirsep/g;
-
+       
     # Want to get relative path of local_dirname within the base_directory
-    # but with URL style slashes.
+    # but with URL style slashes. 
     my $local_dirname = &util::filename_within_directory_url_format($dirname, $base_dir);
 
@@ -460,6 +482,6 @@
 	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
 
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
-	my $raw_full_filename = &util::filename_cat($this_file_base_dir, $raw_file_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
+	my $raw_full_filename = &FileUtils::filenameConcatenate($this_file_base_dir, $raw_file_subfile);
 
 	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
@@ -512,5 +534,5 @@
 	    # Re-read the files in the directory to see if there are any new files
 	    last if (!opendir (DIR, $dirname));
-	    my @dirnow = readdir (DIR);
+	    my @dirnow = sort readdir (DIR);
 	    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dirnow;
 	    closedir (DIR);
@@ -540,8 +562,11 @@
 	my $this_file_base_dir = $base_dir;
 	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
-
-	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	# get the canonical unicode version of the filename. This may not match
+	# the filename on the file system. We will use it to compare to regex
+	# in the metadata table.
+	my $unicode_subfile = &util::raw_filename_to_unicode($dirname, $raw_subfile);
+	my $raw_file_subfile = &FileUtils::filenameConcatenate($file, $raw_subfile);
 	my $raw_full_filename 
-	    = &util::filename_cat($this_file_base_dir,$raw_file_subfile);
+	    = &FileUtils::filenameConcatenate($this_file_base_dir,$raw_file_subfile);
 
 	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
@@ -549,9 +574,9 @@
 	    next;
 	}
-	#print STDERR "processing $raw_full_filename\n";
+	print STDERR "** DirectoryPlugin processing $raw_full_filename\n";
 	# Follow Windows shortcuts
-	if ($raw_subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
+	if ($raw_subfile =~ m/(?i)\.lnk$/ && (($ENV{'GSDLOS'} =~ m/^windows$/i) && ($^O ne "cygwin"))) {
 	    require Win32::Shortcut;
-	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $raw_subfile));
+	    my $shortcut = new Win32::Shortcut(&FileUtils::filenameConcatenate($dirname, $raw_subfile));
 	    if ($shortcut) {
 		# The file to be processed is now the target of the shortcut
@@ -592,13 +617,10 @@
 	}
 
-	# $subfile by this point is url-encoded => all ASCII chars => no need to encode as UTF8
-
-	# Next add metadata read in XML files (if it is supplied)
+	### Now we need to look up the metadata table to see if there is any 
+	# extra metadata for us. We need the canonical unicode version here.
 	if ($additionalmetadata == 1) {
 	    foreach my $filespec (@extrametakeys) {
-		## use the url-encoded filename to do the filename comparison
-
-		if ($subfile =~ /^$filespec$/) {
-		    print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
+		if ($unicode_subfile =~ /^$filespec$/) {
+		    print $outhandle "File \"$unicode_subfile\" matches filespec \"$filespec\"\n" 
 			if ($verbosity > 2);
 		    my $mdref = &extrametautil::getmetadata(\%extrametadata, $filespec);
@@ -624,11 +646,14 @@
 	if (defined $self->{'inf_timestamp'}) {
 	    # Look to see if it's a completely new file
+
 	    if (!$block_hash->{'new_files'}->{$raw_full_filename}) {
 		# Not a new file, must be an existing file
 		# Let' see if it's newer than the last import.pl
-		if (! &util::dir_exists($raw_full_filename)) {
+
+
+		if (! -d $raw_full_filename) {
 		    if (!$block_hash->{'reindex_files'}->{$raw_full_filename}) {
 			# filename has been around for longer than inf_timestamp
-			print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
+			print $outhandle "**** Skipping $unicode_subfile\n" if ($verbosity >3);
 			next;
 		    }
@@ -648,5 +673,5 @@
 
 	# Recursively read each $subfile
-	print $outhandle "DirectoryPlugin recurring: $subfile\n" if ($verbosity > 2);
+	print $outhandle "DirectoryPlugin recurring: $unicode_subfile\n" if ($verbosity > 2);
 	
 	$count += &plugin::read ($pluginfo, $this_file_base_dir,
@@ -663,3 +688,53 @@
 }
 
+# Manifest files, version 2, provide an explicit listing of the documents to be
+# processed by Greenstone.  This allows a user to avoid expensive file tree
+# searches - a crucial requirement for very-large scale collections and
+# parallel processing. However, we still want to leverage the metadata parsing
+# functionality found here in DirectoryPlugin. Thus we have this special call
+# to read that expects a single file. The normal read function starts by
+# listing the files in a given directory and then performs a number of actions
+# over them (including recursing down into any further directories found). We
+# circumvent that behaviour by 'pretending' to already have a directory listing
+# containing at most two file - the file passed in, and an accompanying
+# metadata.xml file if one exists.
+sub read_for_manifest_v2
+{
+    my $self = shift (@_);
+    my ($pluginfo, $file, $block_hash, $processor, $gli) = @_;
+    my $base_dir = '';
+    my $in_metadata = {};
+    my $maxdocs = -1;
+    my $total_count = 0;
+    # Ensure we have the full path of the file to process
+    my $full_path = $file;
+    if ($base_dir =~ /\w/)
+    {
+	$full_path = &FileUtils::filenameConcatenate($base_dir, $file);
+    }
+    # Unlike the vanilla read(), directories are unacceptable
+    if (!-f $full_path)
+    {
+	return 0;
+    }
+    # Now split the full path into a directory and a filename
+    my ($dirname, $the_file) = $full_path =~ /^(.*)\/([^\/]+)$/;
+    # We will prepopulate a 'directory listing' with this file
+    my @dir = ($the_file);
+    # See if there is an accompanying
+    my $metadata_xml_path = $dirname . '/metadata.xml';
+    if (-f $metadata_xml_path)
+    {
+	unshift(@dir, 'metadata.xml');
+    }
+    # Chain through to the normal read process, but with out 'forged' directory
+    # listing so as to avoid all the costs of actually listing / recursing.
+    my $count = $self->read_phase2($pluginfo, $dirname, \@dir, $base_dir, $dirname, $block_hash, $in_metadata, $processor, $maxdocs, $total_count, $gli);
+    # We don't return count, but test that it is 1 exactly.
+    if ($count != 1)
+    {
+	print STDERR "ERROR! The count of documents processed from a single call to DirectoryPlugin::read_for_manifest_v2() is not 1.\n";
+    }
+}
+
 1;
