Changeset 7686 for trunk/gsdl/perllib/plugins/RecPlug.pm

Timestamp:

2004-07-01T14:48:55+12:00 (20 years ago)

Author:

mdewsnip

Message:

First cut at upgrading the CDS/ISIS plugin to obtain and index documents specified in the database (for the UNESCO contract).

File:

• trunk/gsdl/perllib/plugins/RecPlug.pm (modified) (5 diffs)

trunk/gsdl/perllib/plugins/RecPlug.pm

@@ -115 +115 @@
     'desc' => "{RecPlug.use_metadata_files}",
     'type' => "flag",
+    'reqd' => "no" },
+      { 'name' => "recheck_directories",
+    'desc' => "{RecPlug.recheck_directories}",
+    'type' => "flag",
     'reqd' => "no" } ];
 
@@ -137 +141 @@
     if (!parsargv::parse(\@_,
              q^use_metadata_files^, \$self->{'use_metadata_files'},
+             q^recheck_directories^, \$self->{'recheck_directories'},
              "allow_extra_options")) {
     print STDERR "\nRecPlug uses an incorrect option.\n";
@@ -232 +237 @@
     @dir = readdir (DIR);
     closedir (DIR);
-    
+
+    # Re-order the files in the list so any directories ending with .all are moved to the end
+    for ($i = scalar(@dir) - 1; $i >= 0; $i--) {
+    if (-d $dir[$i] && $dir[$i] =~ /\.all$/) {
+        push(@dir, splice(@dir, $i, 1));
+    }
+    }
+
     # read XML metadata files (if supplied)
     my $additionalmetadata = 0;      # is there extra metadata available?
@@ -252 +264 @@
     # import each of the files in the directory
     my $out_metadata; 
-    foreach $subfile (@dir) {
-        
+    my $num_files = scalar(@dir);
+    for (my $i = 0; $i <= scalar(@dir); $i++) {
+    # When every file in the directory has been done, pause for a moment (figuratively!)
+    # If the -recheck_directories argument hasn't been provided, stop now (default)
+    # Otherwise, re-read the contents of the directory to check for new files
+    #   Any new files are added to the @dir list and are processed as normal
+    #   This is necessary when documents to be indexed are specified in bibliographic DBs
+    #   These files are copied/downloaded and stored in a new folder at import time
+    if ($i == $num_files) {
+        last unless $self->{'recheck_directories'};
+
+        # Re-read the files in the directory to see if there are any new files
+        last if (!opendir (DIR, $dirname));
+        my @dirnow = readdir (DIR);
+        closedir (DIR);
+
+        # We're only interested if there are more files than there were before
+        last if (scalar(@dirnow) <= scalar(@dir));
+
+        # Any new files are added to the end of @dir to get processed by the loop
+        foreach my $subfilenow (@dirnow) {
+        for ($j = 0; $j < $num_files; $j++) {
+            last if ($subfilenow eq $dir[$j]);
+        }
+        if ($j == $num_files) {
+            # New file
+            push(@dir, $subfilenow);
+        }
+        }
+
+        # When the new files have been processed, check again
+        $num_files = scalar(@dir);
+    }
+
+    my $subfile = $dir[$i];
     last if ($maxdocs != -1 && $count >= $maxdocs);
     next if ($subfile =~ /^\.\.?$/);
@@ -304 +349 @@
                  $out_metadata, $processor, $maxdocs, $gli);
     }
-    return $count;
-    
+
+    return $count;    
 }