Changeset 27513 for main/trunk

Timestamp:

2013-05-30T10:58:59+12:00 (11 years ago)

Author:

jmt12

Message:

Restoring the original logic around working_info (although still not sure what this does), removing limit of 256 on incremental directory generation (as Kathy points out some MARC collections have thousands of documents all with the same HASH), and had a good go at commenting the function especially the new stuff I've added

File:

• main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm (modified) (1 diff)

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -610 +610 @@
 }
 
-# we now create the directory as part of this call, to try and avoid race
-# conditions caused by parallel processing [jmt12]
-sub get_new_doc_dir{
-   my $self = shift (@_);  
-   my($working_info,$working_dir,$OID) = @_;      
-   
-   
-   my $doc_dir = "";
-   my $doc_dir_rest = $OID;
-
-   # remove any \ and / from the OID
-   $doc_dir_rest =~ s/[\\\/]//g;
-
-   # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
-   $doc_dir_rest =~ s/\://g if ($ENV{'GSDLOS'} =~ /^windows$/i);
-
-   my $doc_dir_num = 0;
-   my $created_directory = 0;
-   do {
-       $doc_dir .= "/" if $doc_dir_num > 0;
-       my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
-       if ($self->{'subdir_hash_prefix'})
-       {
-         $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
-       }
-       #if ($doc_dir_rest =~ s/^(.{1,$limit})//) {
-       if ($doc_dir_rest =~ s/$pattern//i)
-       {
-       $doc_dir .= $1;
-       $doc_dir_num++;
-       }
-       my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir');
-       $created_directory = &FileUtils::makeAllDirectories($new_dir);
-       #rint STDERR "[DEBUG] create directory: $new_dir => $created_directory\n";
-       #rint STDERR "[DEBUG] rest: $doc_dir_rest\n";
-       #rint STDERR "[DEBUG] working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n";
-       #rint STDERR "[DEBUG] doc_dir_num: " . $doc_dir_num . "\n";
-   } while ($doc_dir_rest ne "" && $created_directory == 0 && $doc_dir_num < 2);
-   # what is working_info, because it's values seems to start at 1?
-   # ($working_info->size() >= 1024)
-   my $i = 1;
-   my $doc_dir_base = $doc_dir;
-   while ($created_directory == 0 && $i < 256) {
-       $doc_dir = "$doc_dir_base-$i";
-       $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
-       $i++;
-   }
-
-   if (!$created_directory)
-   {
-     die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
-   }
-
-   return "$doc_dir.dir";
-}
+## @function get_new_doc_dir()
+#
+# Once a doc object is ready to write to disk (and hence has a nice OID),
+# generate a unique subdirectory to write the information to.
+# - create the directory as part of this call, to try and avoid race conditions
+#   found in parallel processing [jmt12]
+#
+sub get_new_doc_dir
+{
+  my $self = shift (@_);
+  my($working_info,$working_dir,$OID) = @_;
+
+  my $doc_dir = "";
+  my $doc_dir_rest = $OID;
+
+  # remove any \ and / from the OID
+  $doc_dir_rest =~ s/[\\\/]//g;
+
+  # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    $doc_dir_rest =~ s/\://g;
+  }
+
+  # we generally create a unique directory by adding consequtive fragments of
+  # the document identifier (split by some predefined length - defaulting to
+  # 8) until we find a directory that doesn't yet exist. Note that directories
+  # that contain a document have a suffix ".dir" (whereas those that contain
+  # only subdirectories have no suffix).
+  my $doc_dir_num = 0; # how many directories deep we are
+  my $created_directory = 0; # have we successfully created a new directory
+  do
+  {
+    # (does this work on windows? - jmt12)
+    if ($doc_dir_num > 0)
+    {
+      $doc_dir .= '/';
+    }
+    # the default matching pattern grabs the next 'subdir_split_length'
+    # characters of the OID to act as the next subdirectory
+    my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
+    # Do we count any "HASH" prefix against the split length limit?
+    if ($self->{'subdir_hash_prefix'} && $doc_dir_num == 0)
+    {
+      $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
+    }
+    # Note the use of 's' to both capture the next chuck of OID and to remove
+    # it from OID at the same time
+    if ($doc_dir_rest =~ s/$pattern//i)
+    {
+      $doc_dir .= $1;
+      $doc_dir_num++;
+      $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
+    }
+    my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir');
+    ###rint STDERR "[DEBUG] BasePlugout::get_new_doc_dir(<working_info>, $working_dir, $oid)\n";
+    ###rint STDERR " - create directory: $new_dir => $created_directory\n";
+    ###rint STDERR " - rest: $doc_dir_rest\n";
+    ###rint STDERR " - working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n";
+    ###rint STDERR " - doc_dir_num: " . $doc_dir_num . "\n";
+  }
+  while ($doc_dir_rest ne '' && ($created_directory == 0 || ($working_info->size() >= 1024 && $doc_dir_num < 2)));
+
+  # not unique yet? Add on an incremental suffix until we are unique
+  my $i = 1;
+  my $doc_dir_base = $doc_dir;
+  while ($created_directory == 0)
+  {
+    $doc_dir = $doc_dir_base . '-' . $i;
+    $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
+    $i++;
+  }
+
+  # in theory this should never happen
+  if (!$created_directory)
+  {
+    die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
+  }
+
+  return $doc_dir . '.dir';
+}
+## get_new_doc_dir()
+
 
 sub process_assoc_files {