Index: /main/trunk/greenstone2/perllib/inexport.pm
===================================================================
--- /main/trunk/greenstone2/perllib/inexport.pm	(revision 23052)
+++ /main/trunk/greenstone2/perllib/inexport.pm	(revision 23053)
@@ -375,5 +375,6 @@
     
     my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
-    if ($self->{'manifest'} ne "") {	
+    if ($self->{'manifest'} ne "") {
+	print STDERR "parsing manifest\n";
 	my $manifest_filename = $self->{'manifest'};
 
@@ -396,8 +397,13 @@
     }
 
+    my $plugin_incr_mode = $incremental_mode;
+    if ($manifest ne "") {
+	# if we have a manifest file, then we pretend we are fully incremental for plugins
+	$plugin_incr_mode = "all";
+    }
     #some global options for the plugins
     my @global_opts = ();
 
-    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $incremental_mode);
+    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $plugin_incr_mode);
     if (scalar(@$pluginfo) == 0) {
 	&gsprintf($out, "{import.no_plugins_loaded}\n");
@@ -424,5 +430,4 @@
 
     # read the archive information file
-##  my $arcinfo_doc_filename = &util::filename_cat ($archivedir, "archives.inf");
 
     # BACKWARDS COMPATIBILITY: Just in case there are old .ldb/.bdb files (won't do anything for other infodbtypes)
@@ -509,13 +514,105 @@
 	&plugin::remove_all($pluginfo, $importdir, $processor, $maxdocs, $gli);
     }
-    if ($manifest eq "") {
-	# process the import directory
-	my $block_hash = {};
-	my $metadata = {};
-	# gobal blocking pass may set up some metadata
-	&plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+
+    # process the import directory
+    my $block_hash = {};
+    $block_hash->{'new_files'} = {};
+    $block_hash->{'reindex_files'} = {};
+    my $metadata = {};
+    
+    # gobal blocking pass may set up some metadata
+    &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+    
+    if ($manifest ne "") {
+	# 
+	# 1. Process delete files first
+	# 
+
+	my @deleted_files = keys %{$manifest_lookup->{'delete'}};
+	my @full_deleted_files = ();
+
+	# ensure all filenames are absolute
+	foreach my $df (@deleted_files) {
+	    #print STDERR "**delete file $df\n";
+	    my $full_df =
+		(&util::filename_is_absolute($df)) 
+		? $df
+		: &util::filename_cat($importdir,$df);
+
+	    if (-d $full_df) {
+		&add_dir_contents_to_list($full_df, \@full_deleted_files);
+	    } else {
+		push(@full_deleted_files,$full_df);
+	    }
+	}
+	
+	&plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
+	mark_docs_for_deletion($archive_info,{},
+			       \@full_deleted_files,
+			       $archivedir, $verbosity, "delete");
+
+
+	# 
+	# 2. Now files for reindexing
+	# 
+
+	my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
+	my @full_reindex_files = ();
+
+	# ensure all filenames are absolute
+	foreach my $rf (@reindex_files) {	    
+	    my $full_rf =
+		(&util::filename_is_absolute($rf)) 
+		? $rf
+		: &util::filename_cat($importdir,$rf);
+
+	    if (-d $full_rf) {
+		&add_dir_contents_to_list($full_rf, \@full_reindex_files);
+	    } else {
+		push(@full_reindex_files,$full_rf);
+	    }
+	}
+	
+	&plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
+	mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
+
+	# And now to ensure the new version of the file processed by 
+	# appropriate plugin, we need to add it to block_hash reindex list
+	foreach my $full_rf (@full_reindex_files) {
+	    #print STDERR "***reindex file $full_rf\n";
+	    $block_hash->{'reindex_files'}->{$full_rf} = 1;
+	}
+
+
+	# 
+	# 3. Now finally any new files - add to block_hash new_files list
+	# 
+
+	my @new_files = keys %{$manifest_lookup->{'index'}};
+	my @full_new_files = ();
+
+	foreach my $nf (@new_files) {
+	    #print STDERR "***index file $nf\n";
+	    # ensure filename is absolute
+	    my $full_nf =
+		(&util::filename_is_absolute($nf)) 
+		? $nf
+		: &util::filename_cat($importdir,$nf);
+
+	    if (-d $full_nf) {
+		&add_dir_contents_to_list($full_nf, \@full_new_files);
+	    } else {
+		push(@full_new_files,$full_nf);
+	    }
+	}
+
+	foreach my $f (@full_new_files) {
+	    $block_hash->{'new_files'}->{$f} = 1;
+	}
+    }
+    else {
+	# if incremental, we read through the import folder to see whats changed.
 
 	if ($incremental || $incremental_mode eq "onlyadd") {
-
 	    prime_doc_oid_count($archivedir);
 
@@ -571,101 +668,22 @@
 		}
 				
-	    }
-	    
-	    # Play it safe, and run through the entire folder, only processing new or edited files
-	    
-            if ((defined $jobs) && ($jobs > 1))
-            { 
-		# if jobs are set to >1, run in parallel using MPI helper
-		# [hs, 1 july 2010]
-		&ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
-						      $self->{'collection'}, $self->{'site'}); 
-            }
-            else
-            {
-	       &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-            }
-	}
-	else {
-            if ((defined $jobs) && ($jobs > 1))
-            { 
-		# if jobs are set to >1, run in parallel using MPI helper
-		# [hs, 1 july 2010]
-		&ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
-						      $self->{'collection'}, $self->{'site'}); 
-            }
-            else
-            {
-		&plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-            }   
-	}
-
+	    }	    
+	}
+    }
+    
+    # now, whichever mode we are in, we can process the entire import folder
+    if ((defined $jobs) && ($jobs > 1))
+    { 
+	# if jobs are set to >1, run in parallel using MPI helper
+	# [hs, 1 july 2010]
+	&ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
+					      $self->{'collection'}, $self->{'site'}); 
     }
     else
     {
-	# 
-	# 1. Process delete files first
-	# 
-
-	my @deleted_files = keys %{$manifest_lookup->{'delete'}};
-	my @full_deleted_files = ();
-
-	# ensure all filenames are absolute
-	foreach my $df (@deleted_files) {	    
-	    my $full_df =
-		(&util::filename_is_absolute($df)) 
-		? $df
-		: &util::filename_cat($importdir,$df);
-
-	    push(@full_deleted_files,$full_df);
-	}
-	
-	&plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
-	mark_docs_for_deletion($archive_info,{},
-			       \@full_deleted_files,
-			       $archivedir, $verbosity, "delete");
-
-
-	# 
-	# 2. Now files for reindexing
-	# 
-
-	my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
-	my @full_reindex_files = ();
-
-	# ensure all filenames are absolute
-	foreach my $rf (@reindex_files) {	    
-	    my $full_rf =
-		(&util::filename_is_absolute($rf)) 
-		? $rf
-		: &util::filename_cat($importdir,$rf);
-
-	    push(@full_reindex_files,$full_rf);
-	}
-	
-	&plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
-	mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
-
-	# And now ensure the new version of the file processed by appropriate
-	# plugin
-	foreach my $full_rf (@full_reindex_files) {
-	    &plugin::read ($pluginfo, "", $full_rf, {}, {}, $processor, $maxdocs, 0, $gli);
-	}
-
-
-	# 
-	# 3. Now finally any new files
-	# 
-
-	foreach my $file (keys %{$manifest_lookup->{'index'}}) {
-	    my $block_hash = {};
-	    my $metadata = {};
-	    &plugin::file_block_read($pluginfo, $importdir, $file, $block_hash, $metadata, $gli);
-	    &plugin::read ($pluginfo, $importdir, $file, $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-	}
-
-
-    }
-
+	&plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    }
+    
+    
     if ($saveas eq "FedoraMETS") {
 	# create collection "doc obj" for Fedora that contains
@@ -1123,4 +1141,31 @@
 }
 
+sub add_dir_contents_to_list {
+
+    my ($dirname, $list) = @_;
+ 
+    # Recur over directory contents.
+    my (@dir, $subfile);
+    
+    # find all the files in the directory
+    if (!opendir (DIR, $dirname)) {
+	print STDERR "inexport: WARNING - couldn't read directory $dirname\n";
+	return -1; # error in processing
+    }
+    @dir = readdir (DIR);
+    closedir (DIR);
+    
+    for (my $i = 0; $i < scalar(@dir); $i++) {
+	my $subfile = $dir[$i];
+	next if ($subfile =~ m/^\.\.?$/);
+	my $full_file = &util::filename_cat($dirname, $subfile);
+	if (-d $full_file) {
+	    &add_dir_contents_to_list($full_file, $list);
+	} else {
+	    push (@$list, $full_file);
+	}
+    }
+	
+}
 
 
