Changeset 26932

Timestamp:

2013-02-26T09:52:47+13:00 (11 years ago)

Author:

jmt12

Message:

Altered all calls to built-in perl file tests to instead use util library ones. This allows better awareness of HDFS or other strange file paths. Added support for newer version of manifest (where files are followed verbatim). Only write OIDcount for numerical OID collections

File:

• gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm (modified) (12 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm

@@ -234 +234 @@
     # fill in the default import and archives directories if none
     # were supplied, turn all \ into / and remove trailing /
-    $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
-    $importdir =~ s/[\\\/]+/\//g;
-    $importdir =~ s/\/$//;
-    if (!-e $importdir) {
+    if ($importdir eq "")
+    {
+      $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import");
+    }
+    else
+    {
+      # hijack filename_cat to sanitize the user provided importdir [jmt12]
+      $importdir = &util::filename_cat ($importdir);
+    }
+    if (!&util::dir_exists($importdir)) {
     &gsprintf($out, "{import.no_import_dir}\n\n", $importdir);
     die "\n";
@@ -256 +262 @@
     }
     }
-
-    $archivedir =~ s/[\\\/]+/\//g;
-    $archivedir =~ s/\/$//;
+    else
+    {
+      # use filename_cat() to sanitize the user provided archive directory as
+      # it is more aware of protocols etc
+      $archivedir = &util::filename_cat($archivedir);
+    }
     $self->{'archivedir'} = $archivedir;
 
@@ -273 +282 @@
     $self->{'manifest'} = $collectcfg->{'manifest'};
     }
+    # Default value
+    $self->{'manifest_version'} = 0;
 
     if (defined $collectcfg->{'gzip'} && !$self->{'gzip'}) {
@@ -427 +438 @@
 
     $manifest_lookup->parse($manifest_filename);
+
+        # Manifests may now include a version number
+        $self->{'manifest_version'} = $manifest_lookup->get_version();
     }
 
@@ -455 +469 @@
 
     if ($removeold) {
-    if (-e $archivedir) {
+    if (&util::dir_exists($archivedir)) {
         &gsprintf($out, "{import.removing_archives}\n");
         &util::rm_r ($archivedir);
@@ -462 +476 @@
     $tmpdir =~ s/[\\\/]+/\//g;
     $tmpdir =~ s/\/$//;
-    if (-e $tmpdir) {
-        #&gsprintf($out, "{import.removing_tmpdir}\n");
-        #&util::rm_r ($tmpdir);
+    if (&util::dir_exists($tmpdir)) {
+        &gsprintf($out, "{import.removing_tmpdir}\n");
+        &util::rm_r ($tmpdir);
     }
     }
@@ -481 +495 @@
     # and attach themselves as a listener (even though they don't do anything)
     # This is done so that, in parallel importing, the server will persist
-    # until the top level import.pl (which will be the first this that calls
-    # this function) completes. [jmt12]
+    # until the top level import.pl (which will be the first that calls this
+    # function) completes. [jmt12]
     my $create_server = 0;
     # - infodb's of type *server need to be started on the same machine that
@@ -562 +576 @@
     }
 
-    my $processor = &plugout::load_plugout($plugout);                        
+    my $processor = &plugout::load_plugout($plugout);
     $processor->setoutputdir ($archivedir);
     $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
@@ -714 +728 @@
 
       # If we are not using complex inherited metadata (and thus have skipped
-      # the global file scan) we need to at least scan the directory of the
-      # files being indexed/reindexed. [jmt12]
-      if (!defined $collectcfg->{'complexmeta'} || $collectcfg->{'complexmeta'} ne 'true')
+      # the global file scan) we need to at least check for a matching
+      # metadata.xml for the files being indexed/reindexed. [jmt12]
+      # - unless we are using the newer version of Manifests, which are treated
+      #   verbatim, and should have a metadata element for metadata files (so
+      #   we can explicitly process metadata files other than metadata.xml)
+      if ($self->{'manifest_version'} < 1 && (!defined $collectcfg->{'complexmeta'} || $collectcfg->{'complexmeta'} ne 'true'))
       {
         my @all_files_to_import = (keys %{$block_hash->{'reindex_files'}}, keys %{$block_hash->{'new_files'}});
         foreach my $file_to_import (@all_files_to_import)
         {
-          my $dir_to_import = $file_to_import;
-          $dir_to_import =~ s/[^\\\/]*$//;
-          # - one day we may need to manually scan this directory for child
-          #   directories and somehow explicitly block them from being
-          #   recursed.
-          if (-d $dir_to_import)
+          my $metadata_xml_path = $file_to_import;
+          $metadata_xml_path =~ s/[^\\\/]*$/metadata.xml/;
+          if (&util::file_exists($metadata_xml_path))
           {
-            &plugin::file_block_read($pluginfo, $dir_to_import, '', $block_hash, $metadata, $gli);
+            &plugin::file_block_read($pluginfo, '', $metadata_xml_path, $block_hash, $metadata, $gli);
           }
+        }
+      }
+
+      if ($self->{'manifest_version'} > 0)
+      {
+        # Process metadata files (?)
+        # Process files
+        foreach my $file_to_import (keys %{$block_hash->{'reindex_files'}}, keys %{$block_hash->{'new_files'}})
+        {
+          &plugin::read ($pluginfo, '', $file_to_import, $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
         }
       }
@@ -823 +847 @@
                           $self->{'collection'}, $self->{'site'}); 
     }
-    else
+    # only do this if we aren't using the newer paradigm for manifest files
+    elsif ($self->{'manifest_version'} < 1)
     {
     &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
@@ -852 +877 @@
     # Store the value of OIDCount (used in doc.pm) so it can be
     # restored correctly to this value on an incremental build
-    store_doc_oid_count($archivedir);
+    # - this OIDcount file should only be generated for numerical oids [jmt12]
+    if ($self->{'OIDtype'} eq 'incremental')
+    {
+      store_doc_oid_count($archivedir);
+    }
 
     # write out the archive information file
@@ -992 +1021 @@
 
 
-    if (open(OIDOUT,">$oid_count_filename")) {
+    if (open(OIDOUT,&util::file_openfdcommand($oid_count_filename, '>'))) {
     print OIDOUT $doc::OIDcount, "\n";