Changeset 27504

Show
Ignore:
Timestamp:
29.05.2013 09:49:08 (6 years ago)
Author:
jmt12
Message:

Changing the wat get_new_doc_dir() works so that it creates the new directories immediately - this is an attempt to solve the race condition issue seen in parallel processing when two document happen to end up with the same hash

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

    r27501 r27504  
    610610} 
    611611 
     612# we now create the directory as part of this call, to try and avoid race 
     613# conditions caused by parallel processing [jmt12] 
    612614sub get_new_doc_dir{ 
    613615   my $self = shift (@_);   
     
    625627 
    626628   my $doc_dir_num = 0; 
    627  
     629   my $created_directory = 0; 
    628630   do { 
    629631       $doc_dir .= "/" if $doc_dir_num > 0; 
     
    639641       $doc_dir_num++; 
    640642       } 
    641    } while ($doc_dir_rest ne "" &&  
    642         ((-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) ||  
    643          ($working_info->size() >= 1024 && $doc_dir_num < 2))); 
     643       my $new_dir = &FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir'); 
     644       $created_directory = &FileUtils::makeAllDirectories($new_dir); 
     645       #rint STDERR "[DEBUG] create directory: $new_dir => $created_directory\n"; 
     646       #rint STDERR "[DEBUG] rest: $doc_dir_rest\n"; 
     647       #rint STDERR "[DEBUG] working_info->size(): " . $working_info->size() . " [ < 1024 ?]\n"; 
     648       #rint STDERR "[DEBUG] doc_dir_num: " . $doc_dir_num . "\n"; 
     649   } while ($doc_dir_rest ne "" && $created_directory == 0 && $doc_dir_num < 2); 
     650   # what is working_info, because it's values seems to start at 1? 
     651   # ($working_info->size() >= 1024) 
    644652   my $i = 1; 
    645653   my $doc_dir_base = $doc_dir; 
    646    while (-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) { 
     654   while ($created_directory == 0 && $i < 256) { 
    647655       $doc_dir = "$doc_dir_base-$i"; 
     656       $created_directory = &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir . '.dir')); 
    648657       $i++; 
    649658   } 
    650            
     659 
     660   if (!$created_directory) 
     661   { 
     662     die("Error! Failed to create directory for document: " . $doc_dir_base . "\n"); 
     663   } 
     664 
    651665   return "$doc_dir.dir"; 
    652666} 
     
    843857        print $rss_fh "   <link>_httpdomain__httpcollection_/document/$oid</link>\n"; 
    844858        print $rss_fh "</item>\n"; 
    845         &FileUtils::closeFileHandle($rss_fh); 
     859        &FileUtils::closeFileHandle($rss_filename, $rss_fh); 
    846860    } 
    847861    else {