Changeset 27504 for main

Timestamp:

2013-05-29T09:49:08+12:00 (11 years ago)

Author:

jmt12

Message:

Changing the wat get_new_doc_dir() works so that it creates the new directories immediately - this is an attempt to solve the race condition issue seen in parallel processing when two document happen to end up with the same hash

File:

• main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm (modified) (4 diffs)

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -610 +610 @@
 }
 
+# we now create the directory as part of this call, to try and avoid race
+# conditions caused by parallel processing [jmt12]
 sub get_new_doc_dir{
    my $self = shift (@_);  
@@ -625 +627 @@
 
    my $doc_dir_num = 0;
-
+   my $created_directory = 0;
    do {
        $doc_dir .= "/" if $doc_dir_num > 0;
@@ -639 +641 @@
        $doc_dir_num++;
        }
-   } while ($doc_dir_rest ne "" && 
-        ((-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) || 
-         ($working_info->size() >= 1024 && $doc_dir_num < 2)));
+       my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir');
+       $created_directory = &FileUtils::makeAllDirectories($new_dir);
+       #rint STDERR "[DEBUG] create directory: $new_dir => $created_directory\n";
+       #rint STDERR "[DEBUG] rest: $doc_dir_rest\n";
+       #rint STDERR "[DEBUG] working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n";
+       #rint STDERR "[DEBUG] doc_dir_num: " . $doc_dir_num . "\n";
+   } while ($doc_dir_rest ne "" && $created_directory == 0 && $doc_dir_num < 2);
+   # what is working_info, because it's values seems to start at 1?
+   # ($working_info->size() >= 1024)
    my $i = 1;
    my $doc_dir_base = $doc_dir;
-   while (-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) {
+   while ($created_directory == 0 && $i < 256) {
        $doc_dir = "$doc_dir_base-$i";
+       $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'));
        $i++;
    }
-          
+
+   if (!$created_directory)
+   {
+     die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
+   }
+
    return "$doc_dir.dir";
 }
@@ -843 +857 @@
         print $rss_fh "   <link>_httpdomain__httpcollection_/document/$oid</link>\n";
         print $rss_fh "</item>\n";
-        &FileUtils::closeFileHandle($rss_fh);
+        &FileUtils::closeFileHandle($rss_filename, $rss_fh);
     }
     else {