Index: main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 27511)
+++ main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 27513)
@@ -610,59 +610,87 @@
 }
 
-# we now create the directory as part of this call, to try and avoid race
-# conditions caused by parallel processing [jmt12]
-sub get_new_doc_dir{
-   my $self = shift (@_);  
-   my($working_info,$working_dir,$OID) = @_;      
-   
-   
-   my $doc_dir = "";
-   my $doc_dir_rest = $OID;
-
-   # remove any \ and / from the OID
-   $doc_dir_rest =~ s/[\\\/]//g;
-
-   # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
-   $doc_dir_rest =~ s/\://g if ($ENV{'GSDLOS'} =~ /^windows$/i);
-
-   my $doc_dir_num = 0;
-   my $created_directory = 0;
-   do {
-       $doc_dir .= "/" if $doc_dir_num > 0;
-       my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
-       if ($self->{'subdir_hash_prefix'})
-       {
-         $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
-       }
-       #if ($doc_dir_rest =~ s/^(.{1,$limit})//) {
-       if ($doc_dir_rest =~ s/$pattern//i)
-       {
-	   $doc_dir .= $1;
-	   $doc_dir_num++;
-       }
-       my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir');
-       $created_directory = &FileUtils::makeAllDirectories($new_dir);
-       #rint STDERR "[DEBUG] create directory: $new_dir => $created_directory\n";
-       #rint STDERR "[DEBUG] rest: $doc_dir_rest\n";
-       #rint STDERR "[DEBUG] working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n";
-       #rint STDERR "[DEBUG] doc_dir_num: " . $doc_dir_num . "\n";
-   } while ($doc_dir_rest ne "" && $created_directory == 0 && $doc_dir_num < 2);
-   # what is working_info, because it's values seems to start at 1?
-   # ($working_info->size() >= 1024)
-   my $i = 1;
-   my $doc_dir_base = $doc_dir;
-   while ($created_directory == 0 && $i < 256) {
-       $doc_dir = "$doc_dir_base-$i";
-       $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
-       $i++;
-   }
-
-   if (!$created_directory)
-   {
-     die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
-   }
-
-   return "$doc_dir.dir";
-}
+## @function get_new_doc_dir()
+#
+# Once a doc object is ready to write to disk (and hence has a nice OID),
+# generate a unique subdirectory to write the information to.
+# - create the directory as part of this call, to try and avoid race conditions
+#   found in parallel processing [jmt12]
+#
+sub get_new_doc_dir
+{
+  my $self = shift (@_);
+  my($working_info,$working_dir,$OID) = @_;
+
+  my $doc_dir = "";
+  my $doc_dir_rest = $OID;
+
+  # remove any \ and / from the OID
+  $doc_dir_rest =~ s/[\\\/]//g;
+
+  # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
+  if ($ENV{'GSDLOS'} =~ /^windows$/i)
+  {
+    $doc_dir_rest =~ s/\://g;
+  }
+
+  # we generally create a unique directory by adding consequtive fragments of
+  # the document identifier (split by some predefined length - defaulting to
+  # 8) until we find a directory that doesn't yet exist. Note that directories
+  # that contain a document have a suffix ".dir" (whereas those that contain
+  # only subdirectories have no suffix).
+  my $doc_dir_num = 0; # how many directories deep we are
+  my $created_directory = 0; # have we successfully created a new directory
+  do
+  {
+    # (does this work on windows? - jmt12)
+    if ($doc_dir_num > 0)
+    {
+      $doc_dir .= '/';
+    }
+    # the default matching pattern grabs the next 'subdir_split_length'
+    # characters of the OID to act as the next subdirectory
+    my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
+    # Do we count any "HASH" prefix against the split length limit?
+    if ($self->{'subdir_hash_prefix'} && $doc_dir_num == 0)
+    {
+      $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
+    }
+    # Note the use of 's' to both capture the next chuck of OID and to remove
+    # it from OID at the same time
+    if ($doc_dir_rest =~ s/$pattern//i)
+    {
+      $doc_dir .= $1;
+      $doc_dir_num++;
+      $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
+    }
+    my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir');
+    ###rint STDERR "[DEBUG] BasePlugout::get_new_doc_dir(<working_info>, $working_dir, $oid)\n";
+    ###rint STDERR " - create directory: $new_dir => $created_directory\n";
+    ###rint STDERR " - rest: $doc_dir_rest\n";
+    ###rint STDERR " - working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n";
+    ###rint STDERR " - doc_dir_num: " . $doc_dir_num . "\n";
+  }
+  while ($doc_dir_rest ne '' && ($created_directory == 0 || ($working_info->size() >= 1024 && $doc_dir_num < 2)));
+
+  # not unique yet? Add on an incremental suffix until we are unique
+  my $i = 1;
+  my $doc_dir_base = $doc_dir;
+  while ($created_directory == 0)
+  {
+    $doc_dir = $doc_dir_base . '-' . $i;
+    $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
+    $i++;
+  }
+
+  # in theory this should never happen
+  if (!$created_directory)
+  {
+    die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
+  }
+
+  return $doc_dir . '.dir';
+}
+## get_new_doc_dir()
+
 
 sub process_assoc_files {
