Changeset 30354

Timestamp:

2015-12-16T16:15:39+13:00 (8 years ago)

Author:

jmt12

Message:

Extending manifest v2 support to allow for directories to be listed in manifest. Matched with changes in Directory plugin to allow paths into systems like HDFS to be listed in manifest.cd

Location:

gs2-extensions/parallel-building/trunk/src

Files:

• bin/script/generate_gantt.pl (modified) (1 diff)
• bin/script/hadoop_import.pl (modified) (4 diffs)
• perllib/FileUtils/HDFSShell.pm (modified) (2 diffs)
• perllib/inexport.pm (modified) (7 diffs)
• src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java (modified) (2 diffs)
• src/java/org/nzdl/gsdl/HadoopGreenstoneIngest2.java (modified) (3 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl

@@ -4 +4 @@
 use strict;
 use warnings;
+
+BEGIN
+{
+    if ( !defined $ENV{'GEXTPARALLELBUILDING_INSTALLED'}) {
+    die "GEXTPARALLELBUILDING_INSTALLED not set\n";
+    }
+    # Installed CPAN packages for GEXT*INSTALL
+    my $perl_version = `perl-version.pl`;
+    my $perl_path = sprintf("%s/lib/perl/%s", $ENV{'GEXTPARALLELBUILDING_INSTALLED'}, $perl_version);
+    unshift (@INC, $perl_path);
+}
 
 # Modules

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

@@ -33 +33 @@
 my $hdfs_fs_prefix = 'HDThriftFS://';
 my $refresh_import = 0;
+my $remove_old = 0;
 my $username = `whoami`;
 chomp($username);
@@ -49 +50 @@
 else
 {
-  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-disable_thrift] [-dry_run] [-start_thrift] [-refresh_import] [-flush_diskcache] [-use_nfs] [-stagger]\n";
+  print STDERR "usage: hadoop_import.pl <collection> [-debug] [-enable_thrift] [-dry_run] [-start_thrift] [-refresh_import] [-flush_diskcache] [-use_nfs] [-stagger] [-removeold]\n";
   print STDERR "where: [debug] print more debug messages to STDERR\n";
   print STDERR "       [dry_run] don't actually perform an file actions\n";
@@ -61 +62 @@
     $debug = 1;
   }
-  if ($ARGV[$offset] eq '-disable_thrift')
-  {
-    $use_thrift = 0;
+  if ($ARGV[$offset] eq '-enable_thrift')
+  {
+    $use_thrift = 1;
   }
   if ($ARGV[$offset] eq '-dry_run')
@@ -88 +89 @@
   {
     $use_nfs = 1;
+  }
+  if ($ARGV[$offset] eq '-removeold')
+  {
+    $remove_old = 1;
   }
   if ($ARGV[$offset] eq '-logdir')

gs2-extensions/parallel-building/trunk/src/perllib/FileUtils/HDFSShell.pm

@@ -249 +249 @@
 }
 ## isHDFS()
+
+
+## @function isSpecialDirectory
+#
+sub isSpecialDirectory
+{
+    my ($path) = @_;
+    return ($path =~ /^HDFSShell:\/\/[a-zA-Z]+:\d+$/);
+}
+## isSpecialDirectory()
 
 
@@ -318 +328 @@
   my @files;
   my $result = &_executeHDFSCommand(1, 'ls', $path);
+  if ($result =~ /No such file or directory/)
+  {
+      print STDERR "BOOM! BOOM! BOOM!\n";
+      return undef;
+  }
   my @lines = split(/\r?\n/, $result);
   foreach my $line (@lines)

gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm

@@ -190 +190 @@
     my $self = { 'xml' => 0, 'mode' => $mode };
 
-    print "INFO: This inexport.pm supports version 2 manifest files\n";
-
     # general options available to all plugins
     my $arguments = $options->{'args'};
@@ -201 +199 @@
     print STDERR "Something went wrong during parsing the arguments. Scroll up for details.\n";
     die "\n";
+    }
+
+    if ($self->{'verbosity'} > 2) {
+    print "[INFO] This inexport.pm supports version 2 manifest files\n";
+    }
+    if ($self->{'verbosity'} > 3) {
+    print '[DEBUG] Perl @INC: ' . join(", ", @INC) . "\n";
     }
 
@@ -735 +740 @@
     else
     {
-      print STDERR "Skipping global file scan due to manifest and complexmeta configuration\n";
+      print STDERR "Skipping import directory-level global file scan due to manifest and complexmeta configuration\n";
     }
 
@@ -757 +762 @@
         : &FileUtils::filenameConcatenate($importdir,$df);
 
-        if (-d $full_df) {
+        if (-d $full_df && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_df, \@full_deleted_files);
         } else {
@@ -783 +788 @@
         : &FileUtils::filenameConcatenate($importdir,$rf);
 
-        if (-d $full_rf) {
+        if (-d $full_rf && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_rf, \@full_reindex_files);
         } else {
@@ -814 +819 @@
         : &FileUtils::filenameConcatenate($importdir,$nf);
 
-        if (-d $full_nf) {
+        if (-d $full_nf && $self->{'manifest_version'} != 2) {
         &add_dir_contents_to_list($full_nf, \@full_new_files);
         } else {
@@ -885 +890 @@
         foreach my $file_to_import (keys %{$block_hash->{'reindex_files'}}, keys %{$block_hash->{'new_files'}})
         {
-        $self->{'directoryplugin'}->read_for_manifest_v2($pluginfo, $file_to_import, $block_hash, $processor, $gli);
+        if (&FileUtils::directoryExists($file_to_import)) {
+#       print "DEBUG: Directory to import: \"" . $file_to_import . "\"\n";
+        &plugin::file_block_read($pluginfo, '', $file_to_import, $block_hash, $metadata, $gli);
+#       print "\n===== BLOCK HASH =====\n";
+#       Dump($block_hash);
+#       print "\n=====            =====\n\n";
+        $self->perform_process_files($manifest, $pluginfo, $importdir, $file_to_import, $block_hash, $metadata, $processor, $maxdocs);
+        }
+        else
+        {
+#       print "DEBUG: File to import: \"" . $file_to_import . "\"\n";
+        $self->{'directoryplugin'}->read_for_manifest_v2($pluginfo, $file_to_import, $block_hash, $processor, $gli);
+        }
         }
       }

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest.java

@@ -262 +262 @@
 
       // - call Greenstone passing in the path to the manifest
-      ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
+      //ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
+      String environment_script_filename = "setup.bash";
+      StringBuffer cmd_buffer = new StringBuffer();
+      cmd_buffer.append("source ./");
+      cmd_buffer.append(environment_script_filename);
+      cmd_buffer.append(" && time -p import.pl -keepold -manifest \"");
+      cmd_buffer.append(manifest_path.toString());
+      cmd_buffer.append("\" -archivedir \"");
+      cmd_buffer.append(conf.get("archivesdir"));
+      cmd_buffer.append("\" ");
+      cmd_buffer.append(collection);
+      ProcessBuilder import_process_builder = new ProcessBuilder("bash", "-c", cmd_buffer.toString());
       fw1.write("[Command:" + import_process_builder.command() + "]\n");
+      /*
       // - alter environment
       Map<String, String> import_process_env = import_process_builder.environment();
-      //   - path
+      // - build up the path
       String path = import_process_env.get("PATH");
       path = gsdlhome + "/ext/parallel-building/bin/script:" + path;
@@ -294 +306 @@
       import_process_env.put("HADOOP_PREFIX", hadoop_home);
       fw1.write("[HADOOP_PREFIX: " + hadoop_home + "]\n");
+      */
 
       // - change working directory

gs2-extensions/parallel-building/trunk/src/src/java/org/nzdl/gsdl/HadoopGreenstoneIngest2.java

@@ -324 +324 @@
       manifest_writer.close();
 
+      /* Original process calling - sets up environment in Java
       // - call Greenstone passing in the path to the manifest
       ProcessBuilder import_process_builder = new ProcessBuilder("time", "-p", "import.pl", "-manifest", manifest_path.toString(), "-keepold", "-archivedir", conf.get("archivesdir"), collection);
@@ -357 +358 @@
       import_process_env.put("HADOOP_PREFIX", hadoop_home);
       fw1.write("[HADOOP_PREFIX: " + hadoop_home + "]\n");
+      */
+
+      /* New process call - adds call to setup.bash first to prepare
+       * environment... hopefully */
+      // - call Greenstone passing in the path to the manifest
+      String environment_script_filename = "setup.bash";
+      StringBuffer cmd_buffer = new StringBuffer();
+      cmd_buffer.append("source ./");
+      cmd_buffer.append(environment_script_filename);
+      cmd_buffer.append(" && time -p import.pl -keepold -manifest \"");
+      cmd_buffer.append(manifest_path.toString());
+      cmd_buffer.append("\" -archivedir \"");
+      cmd_buffer.append(conf.get("archivesdir"));
+      cmd_buffer.append("\" ");
+      cmd_buffer.append(collection);
+      ProcessBuilder import_process_builder = new ProcessBuilder("bash", "-c", cmd_buffer.toString());
+      fw1.write("[Command:" + import_process_builder.command() + "]\n");
 
       // - change working directory
@@ -738 +756 @@
     job.setReducerClass(GSReducer.class);
 
-    // Sets the input and output handlers - may need to adjust input to provide me
-    // a series of filenames (TextInputFormat will instead read in a text file and
-    // return each line...)
+    // Sets the input and output handlers - may need to adjust input to provide
+    // a series of filenames (TextInputFormat will instead read in a text file
+    // and return each line...)
     job.setInputFormatClass(GSFileInputFormat.class);
     //job.setOutputFormatClass(NullOutputFormat.class);