Changeset 30354 for gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm

Timestamp:

2015-12-16T16:15:39+13:00 (8 years ago)

Author:

jmt12

Message:

Extending manifest v2 support to allow for directories to be listed in manifest. Matched with changes in Directory plugin to allow paths into systems like HDFS to be listed in manifest.cd

File:

• gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm (modified) (7 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm

@@ -190 +190 @@
     my $self = { 'xml' => 0, 'mode' => $mode };
 
-    print "INFO: This inexport.pm supports version 2 manifest files\n";
-
     # general options available to all plugins
     my $arguments = $options->{'args'};
@@ -201 +199 @@
     print STDERR "Something went wrong during parsing the arguments. Scroll up for details.\n";
     die "\n";
+    }
+
+    if ($self->{'verbosity'} > 2) {
+    print "[INFO] This inexport.pm supports version 2 manifest files\n";
+    }
+    if ($self->{'verbosity'} > 3) {
+    print '[DEBUG] Perl @INC: ' . join(", ", @INC) . "\n";
     }
 
@@ -735 +740 @@
     else
     {
-      print STDERR "Skipping global file scan due to manifest and complexmeta configuration\n";
+      print STDERR "Skipping import directory-level global file scan due to manifest and complexmeta configuration\n";
     }
 
@@ -757 +762 @@
         : &FileUtils::filenameConcatenate($importdir,$df);
 
-        if (-d $full_df) {
+        if (-d $full_df && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_df, \@full_deleted_files);
         } else {
@@ -783 +788 @@
         : &FileUtils::filenameConcatenate($importdir,$rf);
 
-        if (-d $full_rf) {
+        if (-d $full_rf && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_rf, \@full_reindex_files);
         } else {
@@ -814 +819 @@
         : &FileUtils::filenameConcatenate($importdir,$nf);
 
-        if (-d $full_nf) {
+        if (-d $full_nf && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_nf, \@full_new_files);
         } else {
@@ -885 +890 @@
         foreach my $file_to_import (keys %{$block_hash->{'reindex_files'}}, keys %{$block_hash->{'new_files'}})
         {
-        $self->{'directoryplugin'}->read_for_manifest_v2($pluginfo, $file_to_import, $block_hash, $processor, $gli);
+        if (&FileUtils::directoryExists($file_to_import)) {
+#       print "DEBUG: Directory to import: \"" . $file_to_import . "\"\n";
+        &plugin::file_block_read($pluginfo, '', $file_to_import, $block_hash, $metadata, $gli);
+#       print "\n===== BLOCK HASH =====\n";
+#       Dump($block_hash);
+#       print "\n=====            =====\n\n";
+        $self->perform_process_files($manifest, $pluginfo, $importdir, $file_to_import, $block_hash, $metadata, $processor, $maxdocs);
+        }
+        else
+        {
+#       print "DEBUG: File to import: \"" . $file_to_import . "\"\n";
+        $self->{'directoryplugin'}->read_for_manifest_v2($pluginfo, $file_to_import, $block_hash, $processor, $gli);
+        }
         }
       }