Changeset 24683 for gs2-extensions

Timestamp:

2011-09-28T13:20:26+13:00 (13 years ago)

Author:

jmt12

Message:

Fixes to process (based predominantly on the new/custom complexmeta and block_hash->{manifest} flags) to allow global scan to listen to manifests

File:

• gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm (modified) (5 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/plugins/DirectoryPlugin.pm

@@ -142 +142 @@
     my $self = shift (@_);
     my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+
 }
 
@@ -372 +373 @@
     my $outhandle = $self->{'outhandle'};
     my $verbosity = $self->{'verbosity'};
-
+    
     # Calculate the directory name and ensure it is a directory and
     # that it is not explicitly blocked.
@@ -465 +466 @@
         next;
     }
+
+        # If we are processing a manifest file, then we don't have to guess at
+        # what metadata source files to process. Only process those that appear
+        # in the metadata_files list in the block_hash. [jmt12]
+        if ($block_hash->{'manifest'} eq 'true' 
+            && !-d $raw_full_filename
+            && !$block_hash->{'metadata_files'}->{$raw_full_filename})
+        {
+          print $outhandle "**** Not in manifest. Ignoring metadata in $subfile\n" if ($verbosity > 3);
+          next;
+        }
+
     
     # Recursively read each $raw_subfile
@@ -627 +640 @@
     }
 
+        # So, here's my problem. I have an empty archives directory, and a 
+        # manifest file that says to add 1 out of 2 documents in the import
+        # directory. The directory scan finds both files, and doc 1 is in
+        # new_files, so that is all good. But doc 2 is also processed, and
+        # without any blocking or other rule, also gets imported.
+        # If we are processing a Manifest then files (not docs) that aren't in
+        # either the new files or reindex files are skipped.
+        # We'll also test for metadata files here, even though they should
+        # have been handled above - just in case we have the weird case where
+        # a file is both a metadata source and a file in its own right. [jmt12]
+        if ($block_hash->{'manifest'} eq 'true' 
+            && !-d $raw_full_filename
+            && !$block_hash->{'new_files'}->{$raw_full_filename}
+            && !$block_hash->{'reindex_files'}->{$raw_full_filename}
+            && !$block_hash->{'metadata_files'}->{$raw_full_filename})
+        {
+          print $outhandle "**** Not in manifest. Skipping $subfile\n" if ($verbosity > 3);
+          next;
+        }
+
     if (defined $self->{'inf_timestamp'}) {
         # Look to see if it's a completely new file
@@ -653 +686 @@
         }
         }
-    }
+        }
 
     # Recursively read each $subfile