Changeset 17290 for gsdl/trunk/perllib/plugins/OAIPlugin.pm

Timestamp:

2008-09-15T15:28:48+12:00 (16 years ago)

Author:

kjdon

Message:

previous changes to get exploding working (using metadata_read) meant that general coll building with this plugin stopped working :-(. So, I think I have fixed that, but it will have broken exploding in the meantime. I'll fix that next, but am going home now

File:

• gsdl/trunk/perllib/plugins/OAIPlugin.pm (modified) (8 diffs)

gsdl/trunk/perllib/plugins/OAIPlugin.pm

@@ -47 +47 @@
     'type' => "regexp",
     'reqd' => "no",
-    'deft' => &get_default_process_exp() }
+    'deft' => &get_default_process_exp() },
+      { 'name' => "xxx",
+    'desc' => "{OAIPlugin.xxx}",
+    'type' => "metadata",
+    'reqd' => "no",
+    'deft' => "gi.Sourcedoc" }
       ];
 
@@ -88 +93 @@
     $self->{'in_metadata_node'} = 0;
     $self->{'rawxml'} = "";
+    $self->{'saved_metadata'} = {};
 }
 
@@ -140 +146 @@
     if ($element eq "metadata") {
     my $textref = \$self->{'metadata_xml'};
-    my $metadata = $self->{'metadata'};
+    #my $metadata = $self->{'metadata'};
+    my $metadata = $self->{'saved_metadata'};
     $self->extract_oai_metadata($textref,$metadata);
 
@@ -169 +176 @@
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
     return undef unless $self->can_process_this_file($filename_full_path);
+#    print STDERR "initial\n";
+#   foreach my $k (keys %$metadata) {
+#   print STDERR "$k=".join (", ", @{$metadata->{$k}})."; ";
+#    }
+#    print STDERR "\n";
     
     my $total_count = 0; # is total count used?
-    if ($self->SUPER::read($pluginfo,$base_dir,$file,$block_hash,$metadata,$processor,$maxdocs,$total_count, $gli)) {
-    # calling "SUPER::read" at this point sets up $metadata
-    # data-structure.  We can then, later, in OAIPlug::read decide
-    # whether this $metadata will stick to an accompanying file,
-    # or else needs a new doc object to be formed that contains
-    # purely metadata
-        
-    $self->{'metadata'} = undef;
+    if (!$self->parse_file($filename_full_path, $file, $gli)) {
+    $self->{'saved_metadata'} = undef;
+    return undef;
+    }
+
+    my $new_metadata = $self->{'saved_metadata'};
+    $self->{'saved_metadata'} = undef;
+    # add the pretty metadata table as metadata
+    my $ppmd_table = $self->{'ppmd_table'};
+    $new_metadata->{'prettymd'} = $ppmd_table;
+    $self->{'ppmd_table'} = undef;
+    
+    print STDERR "after parse\n";
+    foreach my $k (keys %$new_metadata) {
+    print STDERR "$k=".join (", ", @{$new_metadata->{$k}})."; ";
+    }
+    print STDERR "\n";
+    
+    
+    #   if ($self->SUPER::read($pluginfo,$base_dir,$file,$block_hash,$new_metadata,$processor,$maxdocs,$total_count, $gli)) {
+    # calling "SUPER::read" at this point sets up $metadata
+    # data-structure.  We can then, later, in OAIPlug::read decide
+    # whether this $metadata will stick to an accompanying file,
+    # or else needs a new doc object to be formed that contains
+    # purely metadata
+    
+#   $self->{'metadata'} = undef;
+#   print STDERR "after erad\n";
+#   foreach my $k (keys %$metadata) {
+#       print STDERR "$k=".join (", ", @{$metadata->{$k}})."; ";
+#   }
+#   print STDERR "\n";
+    my $url_array = $new_metadata->{'dc.Identifier'};
+    my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
+    print STDERR "$num_urls urls for $file\n";
+    my $srcdoc_exists = 0;
+    my $srcdoc_pos = 0;
+    my $filename_dir = &util::filename_head($filename_full_path);
+    my $filename_for_metadata = $file;
+    for (my $i=0; $i<$num_urls; $i++) {
     
-    #my $url_array = $metadata->{'gi.Sourcedoc'};
-    my $url_array = $metadata->{'dc.Identifier'};
-    my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
-
-    my $srcdoc_exists = 0;
-    my $srcdoc_pos = 0;
-    my $filename_dir = &util::filename_head($filename_full_path);
-        
-    for (my $i=0; $i<$num_urls; $i++) {
-        
-        if ($url_array->[$i] !~ m/^(https?|ftp):/) {
-        
-        my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
-        
-        if (-e $src_filename) {
-            $srcdoc_pos = $i;
-            $srcdoc_exists = 1;
-            last;
-        }
+    if ($url_array->[$i] !~ m/^(https?|ftp):/) {
+        
+        my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
+        
+        if (-e $src_filename) {
+        $srcdoc_pos = $i;
+        $srcdoc_exists = 1;
+        $filename_for_metadata = $url_array->[$i];
+        last;
         }
     }
-    
-    
-    if ($srcdoc_exists)
-    {
-        $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
-        
+    }
+    
+    
+    if ($srcdoc_exists)
+    {
+    $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
+    }
+    else {
+    # save the rawxml for the source document
+    $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 0;
+    $self->{'oai-files'}->{$file}->{'rawxml'} = $self->{'rawxml'};
+    $self->{'rawxml'} = "";
+    print STDERR "raw xml = $self->{'oai-files'}->{$file}->{'rawxml'}\n";
+    }
+    
 ###     print STDERR "**** storing OAI file: $file\n";
-        
-        # Make pretty print metadata table stick with src filename
-        my $ppmd_table = $self->{'ppmd_table'};
-        
-        $metadata->{'prettymd'} = [ $ppmd_table ];
-        $self->{'ppmd_table'} = undef;
-        
-    }
-    else {
-        $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 0;
-    }
-    
-    }
-    else {
-    return undef;
-    }
+    
+    # return all the metadata we have extracted to the caller.
+    # Directory plug will pass it back in at read time, so we don't need to extract it again.
+    $extrametadata->{$filename_for_metadata} = $new_metadata;
+    push(@$extrametakeys, $filename_for_metadata);
+
+    return 1;
+    
 }
 
@@ -229 +264 @@
 sub read {
     my $self = shift (@_);  
-  
+    
     my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
 
@@ -235 +270 @@
 ###    print STDERR "**** checking OAI read: $file\n";
 
-    if (defined $self->{'oai-files'}->{$file}) {
-    
-    my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
-
+    if (!defined $self->{'oai-files'}->{$file}) {
+    return undef;
+    }
+    
+    
+    my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
+    if ($srcdoc_exists) {
+    # do nothing more
     # no more need to access details of this $file => tidy up as you go
     delete $self->{'oai-files'}->{$file};
+    return 0; # not processed here, but don't pass on to rest of plugins
+    }
 
 ### print STDERR "**** !!!!! srcdoc_exists = $srcdoc_exists\n"; 
-    if (!$srcdoc_exists)
-    {
-
-        my $filename = $file;
-        $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
-
-        # Do encoding stuff on metadata
-        my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
-
-        # create a new document
-        my $doc_obj = new doc ($filename, "indexed_doc");
-        my $top_section = $doc_obj->get_top_section;
-        my $plugin_type = $self->{'plugin_type'};
-        
-        $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-        $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
-        $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
-        $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
-        $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
-        
-        # include any metadata passed in from previous plugins 
-        # note that this metadata is associated with the top level section
-        $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
-        
-        # do plugin specific processing of doc_obj
-        my $textref = \$self->{'rawxml'};
-        unless (defined ($self->process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
-        print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-        return -1;
-        }
-        
-        # do any automatic metadata extraction
-        $self->auto_extract_metadata ($doc_obj);
-        
-        # add an OID
-        $self->add_OID($doc_obj);
-        
-        my $prettymds = $self->{'prettymd'};
-        foreach my $prettymd (@$prettymds) {
-        $doc_obj->add_utf8_metadata($top_section,"prettymd",$prettymd);
-        }
-        $self->{'prettymd'} = undef;
-        
-        # process the document
-        $processor->process($doc_obj);
-        
-        $self->{'num_processed'} ++;
-        
-        return 1; # processed the file
-    }
-    }
-    else {
-    return undef;
-    }
+    
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+    # Do encoding stuff on metadata
+    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+
+    # create a new document
+    my $doc_obj = new doc ($filename, "indexed_doc");
+    my $top_section = $doc_obj->get_top_section;
+    my $plugin_type = $self->{'plugin_type'};
+    
+    $doc_obj->add_utf8_metadata($top_section, "Language", $language);
+    $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+    $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
+    $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
+    $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
+    
+    # include any metadata passed in from previous plugins 
+    # note that this metadata is associated with the top level section
+    $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+    
+    # do plugin specific processing of doc_obj
+    print STDERR "raw xml 2 = $self->{'oai-files'}->{$file}->{'rawxml'}\n";
+    my $text = $self->{'oai-files'}->{$file}->{'rawxml'};
+    delete $self->{'oai-files'}->{$file};
+
+    unless (defined ($self->process(\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+    print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+    return -1;
+    }
+    
+    # do any automatic metadata extraction
+    $self->auto_extract_metadata ($doc_obj);
+    
+    # add an OID
+    $self->add_OID($doc_obj);
+        
+    # process the document
+    $processor->process($doc_obj);
+    
+    $self->{'num_processed'} ++;
+    
+    return 1; # processed the file
 }
 
@@ -302 +333 @@
 sub read_old {
     my $self = shift (@_);  
-  
+    
     my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
 
@@ -436 +467 @@
     my $style = "style=\'border-bottom: 4px solid #000080\'";
 
-    $self->{'ppmd_table'} = "\n<table $att $style>";
+    $self->{'ppmd_table'} = "\n<table $att $style>";
 }