Changeset 19493 for gsdl

Timestamp:

2009-05-18T10:57:05+12:00 (15 years ago)

Author:

davidb

Message:

Introduction of new extrametafile to track which metadata.xml file a piece of metadata came from

Location:

gsdl/trunk/perllib/plugins

Files:

• ArchivesInfPlugin.pm (modified) (1 diff)
• BasePlugin.pm (modified) (1 diff)
• DSpacePlugin.pm (modified) (1 diff)
• DirectoryPlugin.pm (modified) (8 diffs)
• LOMPlugin.pm (modified) (1 diff)
• MetadataCSVPlugin.pm (modified) (1 diff)
• MetadataEXIFPlugin.pm (modified) (1 diff)
• MetadataXMLPlugin.pm (modified) (5 diffs)
• OAIPlugin.pm (modified) (1 diff)
• PDFPlugin.pm (modified) (1 diff)
• SplitTextFile.pm (modified) (2 diffs)

gsdl/trunk/perllib/plugins/ArchivesInfPlugin.pm

@@ -169 +169 @@
 sub metadata_read {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     return undef;

gsdl/trunk/perllib/plugins/BasePlugin.pm

@@ -906 +906 @@
 sub metadata_read {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
     
     # can we process this file??

gsdl/trunk/perllib/plugins/DSpacePlugin.pm

@@ -240 +240 @@
 sub metadata_read {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     my $only_first_doc = $self->{'only_first_doc'};

gsdl/trunk/perllib/plugins/DirectoryPlugin.pm

@@ -326 +326 @@
 sub metadata_read {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     return undef;
@@ -395 +397 @@
     my $additionalmetadata = 0;      # is there extra metadata available?
     my %extrametadata;               # maps from filespec to extra metadata keys
+    my %extrametafile;               # maps from filespec to the metadata.xml (or similar) file it came from
     my @extrametakeys;               # keys of %extrametadata in order read
 
@@ -411 +414 @@
         my $extrakeys_re  = $ek->{'re'};
         my $extrakeys_md  = $ek->{'md'};
+        my $extrakeys_mf  = $ek->{'mf'};
         push(@extrametakeys,$extrakeys_re);
         $extrametadata{$extrakeys_re} = $extrakeys_md;
+        $extrametafile{$extrakeys_re} = $extrakeys_mf;
     }
     delete($self->{'subdir_extrametakeys'}->{$local_dirname});
@@ -437 +442 @@
                       $file_subfile,$block_hash,
                       \@extrametakeys, \%extrametadata,
+                      \%extrametafile,
                       $processor, $maxdocs, $gli);
     $additionalmetadata = 1;
@@ -455 +461 @@
     if ($ek_non_re =~ m/$dirsep_re/) { # specifies at least one directory
         my $md = $extrametadata{$ek};
+        my $mf = $extrametafile{$ek};
 
         my $subdir_extrametakeys = $self->{'subdir_extrametakeys'};
 
-        my $subdir_rec = { 're' => $subdir_re, 'md' => $md };
+        my $subdir_rec = { 're' => $subdir_re, 'md' => $md, 'mf' => $mf };
 
         # when its looked up, it must be relative to the base dir
@@ -562 +569 @@
     # Next add metadata read in XML files (if it is supplied)
     if ($additionalmetadata == 1) {
-        my ($filespec, $mdref);
-        foreach $filespec (@extrametakeys) {
+        foreach my $filespec (@extrametakeys) {
         ## use the utf8 encoded filename to do the filename comparison
         if ($tmpfile =~ /^$filespec$/) {
             print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
             if ($verbosity > 2);
-            $mdref = $extrametadata{$filespec};
+            my $mdref = $extrametadata{$filespec};
+            my $mfref = $extrametafile{$filespec};
+
+            # Add the list files where the metadata came from
+            # into the metadata table so we can track this
+            # This mechanism is similar to how gsdlassocfile works
+
+            my @metafile_pair = ();
+            foreach my $l (keys %$mfref) {
+            my $f = $mfref->{$l};
+            push (@metafile_pair, "$f:$l");
+            }
+
+            $mdref->{'gsdlmetafile'} = \@metafile_pair;
+
             &metadatautil::combine_metadata_structures($out_metadata, $mdref);
         }
@@ -583 +603 @@
 
 
-        my $inf_timestamp = $self->{'inf_timestamp'};
-
         if (! -d $full_filename) {
-            my $filename_timestamp = -M $full_filename;
-            if ($filename_timestamp > $inf_timestamp) {
+            if (!$block_hash->{'reindex_files'}->{$full_filename}) {
             # filename has been around for longer than inf
             print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
@@ -602 +619 @@
             # mark to doc-oids that rely on it for re-indexing
             ## &GDBMUtils::gdbmDatabase();
-
             }
         }

gsdl/trunk/perllib/plugins/LOMPlugin.pm

@@ -118 +118 @@
 sub metadata_read {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     my $outhandle = $self->{'outhandle'};

gsdl/trunk/perllib/plugins/MetadataCSVPlugin.pm

@@ -79 +79 @@
 {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     # Read metadata from CSV files

gsdl/trunk/perllib/plugins/MetadataEXIFPlugin.pm

@@ -168 +168 @@
 {
   my $self = shift (@_);
-  my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+  my ($pluginfo, $base_dir, $file, $block_hash, 
+      $extrametakeys, $extrametadata, $extrametafile,
+      $processor, $maxdocs, $gli) = @_;
   
 

gsdl/trunk/perllib/plugins/MetadataXMLPlugin.pm

@@ -167 +167 @@
 {
     my $self = shift (@_);
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata,
-$processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata,$extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     my $filename = &util::filename_cat($base_dir, $file);
@@ -174 +175 @@
         return undef;
     }
+
+    $self->{'metadata-file'} = $file;
+    $self->{'metadata-filename'} = $filename;
 
     my $outhandle = $self->{'outhandle'};
@@ -182 +186 @@
     $block_hash->{'file_blocks'}->{$filename} = 1;
     $self->{'metadataref'} = $extrametadata;
+    $self->{'metafileref'} = $extrametafile;
     $self->{'metakeysref'} = $extrametakeys;
     
@@ -236 +241 @@
         my $file_metadata = $self->{'metadataref'}->{$target};
         my $saved_metadata = $self->{'saved_metadata'};
+
         if (!defined $file_metadata) {
         $self->{'metadataref'}->{$target} = $saved_metadata;
@@ -245 +251 @@
         &metadatautil::combine_metadata_structures($file_metadata,$saved_metadata);
         }
+
+        
+        # now record which metadata.xml file it came from
+
+        my $file = $self->{'metadata-file'};
+        my $filename = $self->{'metadata-filename'};
+
+        if (!defined $self->{'metafileref'}->{$target}) {
+        $self->{'metafileref'}->{$target} = {};
+        }
+
+        $self->{'metafileref'}->{$target}->{$file} = $filename
     }
     }

gsdl/trunk/perllib/plugins/OAIPlugin.pm

@@ -183 +183 @@
     my $self = shift (@_);  
 
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     # can we process this file??

gsdl/trunk/perllib/plugins/PDFPlugin.pm

@@ -148 +148 @@
         push(@$pagedimg_options, "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
         push(@$pagedimg_options, "-screenviewsize", "1000");
+        push(@$pagedimg_options, "-enable_cache");
     }
     }

gsdl/trunk/perllib/plugins/SplitTextFile.pm

@@ -118 +118 @@
 sub metadata_read {
     my $self = shift (@_);  
-    my ($pluginfo, $base_dir, $file, $block_hash, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+    my ($pluginfo, $base_dir, $file, $block_hash, 
+    $extrametakeys, $extrametadata, $extrametafile,
+    $processor, $maxdocs, $gli) = @_;
 
     # returns 1 if matches process_exp, and has done blocking in the meantime
@@ -124 +126 @@
                           $block_hash, 
                           $extrametakeys, 
-                          $extrametadata, $processor, 
-                          $maxdocs, $gli);
+                          $extrametadata, 
+                          $extrametafile,
+                          $processor, $maxdocs, $gli);
     my $split_matched = undef;