Changeset 22445

Timestamp:

2010-07-20T13:58:05+12:00 (14 years ago)

Author:

davidb

Message:

Incorporation of top-level parallel_import functionality into code base.

Location:

main/trunk/greenstone2/perllib

Files:

• ParallelInexport.pm (added)
• inexport.pm (modified) (4 diffs)

main/trunk/greenstone2/perllib/inexport.pm

@@ -118 +118 @@
     $self->{'collection'} = shift @$argv;
 
+    if ((defined $self->{'jobs'}) && ($self->{'jobs'}>1)) {
+    require ParallelInexport;
+    }
+
     return bless $self, $class;
 }
@@ -350 +354 @@
 
     my $gli          = $self->{'gli'};
+
+    my $jobs         = $self->{'jobs'};
+    my $epoch        = $self->{'epoch'};
 
     # related to export
@@ -566 +573 @@
         
         # Play it safe, and run through the entire folder, only processing new or edited files
-        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-
+        
+            if ((defined $jobs) && ($jobs > 1))
+            { 
+        # if jobs are set to >1, run in parallel using MPI helper
+        # [hs, 1 july 2010]
+        &ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
+                              $self->{'collection'}, $self->{'site'}); 
+            }
+            else
+            {
+           &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+            }
     }
     else {
-        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+            if ((defined $jobs) && ($jobs > 1))
+            { 
+        # if jobs are set to >1, run in parallel using MPI helper
+        # [hs, 1 july 2010]
+        &ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
+                              $self->{'collection'}, $self->{'site'}); 
+            }
+            else
+            {
+        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+            }   
     }
 
@@ -685 +712 @@
     my $inexport_mode = $self->{'mode'};
 
-    my $statsfile = $self->{'statsfile'};
-    my $out       = $self->{'out'};
+    my $statsfile   = $self->{'statsfile'};
+    my $out         = $self->{'out'};
     my $faillogname = $self->{'faillogname'};
-    my $gli       = $self->{'gli'};
+    my $gli         = $self->{'gli'};
+    my $jobs        = $self->{'jobs'};
 
     # write out import stats
-    my $close_stats = 0;
-    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
-    if (open (STATS, ">$statsfile")) {
-        $statsfile = 'import::STATS';
-        $close_stats = 1;
-    } else {
-        &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
-        &gsprintf($out, "{import.stats_backup}\n");
-        $statsfile = 'STDERR';
-    }
-    }
-
-    &gsprintf($out, "\n");
-    &gsprintf($out, "*********************************************\n");
-    &gsprintf($out, "{$inexport_mode.complete}\n");
-    &gsprintf($out, "*********************************************\n");
-
-    &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
-    if ($close_stats) {
-    close STATS;
+
+    if ((!defined $jobs) || ($jobs == 1))
+    {
+    # only output statistics if there are multiple jobs
+    # [hs, 1 july 2010]
+
+    my $close_stats = 0;
+    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
+        if (open (STATS, ">$statsfile")) {
+        $statsfile = 'import::STATS';
+        $close_stats = 1;
+        } else {
+        &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
+        &gsprintf($out, "{import.stats_backup}\n");
+        $statsfile = 'STDERR';
+        }
+    }
+    
+    &gsprintf($out, "\n");
+    &gsprintf($out, "*********************************************\n");
+    &gsprintf($out, "{$inexport_mode.complete}\n");
+    &gsprintf($out, "*********************************************\n");
+    
+    &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
+    if ($close_stats) {
+        close STATS;
+    }
     }