Changeset 17319 for gsdl/trunk/perllib/plugins/OAIPlugin.pm

Timestamp:

2008-09-18T10:03:44+12:00 (16 years ago)

Author:

kjdon

Message:

tidied this up and removed some old code

File:

• gsdl/trunk/perllib/plugins/OAIPlugin.pm (modified) (14 diffs)

gsdl/trunk/perllib/plugins/OAIPlugin.pm

@@ -48 +48 @@
     'reqd' => "no",
     'deft' => &get_default_process_exp() },
-      { 'name' => "xxx",
-    'desc' => "{OAIPlugin.xxx}",
+      { 'name' => "document_field",
+    'desc' => "{OAIPlugin.document_field}",
     'type' => "metadata",
     'reqd' => "no",
@@ -176 +176 @@
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
     return undef unless $self->can_process_this_file($filename_full_path);
-#    print STDERR "initial\n";
-#   foreach my $k (keys %$metadata) {
-#   print STDERR "$k=".join (", ", @{$metadata->{$k}})."; ";
-#    }
-#    print STDERR "\n";
-    
-    my $total_count = 0; # is total count used?
+    
     if (!$self->parse_file($filename_full_path, $file, $gli)) {
     $self->{'saved_metadata'} = undef;
@@ -190 +184 @@
     my $new_metadata = $self->{'saved_metadata'};
     $self->{'saved_metadata'} = undef;
+
     # add the pretty metadata table as metadata
     my $ppmd_table = $self->{'ppmd_table'};
     $new_metadata->{'prettymd'} = $ppmd_table;
     $self->{'ppmd_table'} = undef;
-    
-    print STDERR "after parse\n";
-    foreach my $k (keys %$new_metadata) {
-    print STDERR "$k=".join (", ", @{$new_metadata->{$k}})."; ";
-    }
-    print STDERR "\n";
-    
-    
-    #   if ($self->SUPER::read($pluginfo,$base_dir,$file,$block_hash,$new_metadata,$processor,$maxdocs,$total_count, $gli)) {
-    # calling "SUPER::read" at this point sets up $metadata
-    # data-structure.  We can then, later, in OAIPlug::read decide
-    # whether this $metadata will stick to an accompanying file,
-    # or else needs a new doc object to be formed that contains
-    # purely metadata
-    
-#   $self->{'metadata'} = undef;
-#   print STDERR "after erad\n";
-#   foreach my $k (keys %$metadata) {
-#       print STDERR "$k=".join (", ", @{$metadata->{$k}})."; ";
-#   }
-#   print STDERR "\n";
-    my $url_array = $new_metadata->{'dc.Identifier'};
+      
+    my $document_metadata_field = $self->{'document_field'};
+    my $url_array = $new_metadata->{$document_metadata_field};
     my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
-    print STDERR "$num_urls urls for $file\n";
+    ##print STDERR "$num_urls urls for $file\n";
     my $srcdoc_exists = 0;
     my $srcdoc_pos = 0;
     my $filename_dir = &util::filename_head($filename_full_path);
-    my $filename_for_metadata = $file;
+    my $filename_for_metadata = $file; # this assumes there will only be one record per oai file - is this always the case??
     for (my $i=0; $i<$num_urls; $i++) {
     
@@ -237 +213 @@
     }
     
-    
-    if ($srcdoc_exists)
-    {
+    if ($srcdoc_exists) {
     $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
     }
@@ -247 +221 @@
     $self->{'oai-files'}->{$file}->{'rawxml'} = $self->{'rawxml'};
     $self->{'rawxml'} = "";
-    print STDERR "raw xml = $self->{'oai-files'}->{$file}->{'rawxml'}\n";
-    }
-    
-###     print STDERR "**** storing OAI file: $file\n";
+    }
     
     # return all the metadata we have extracted to the caller.
@@ -267 +238 @@
     my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
 
-
-###    print STDERR "**** checking OAI read: $file\n";
-
     if (!defined $self->{'oai-files'}->{$file}) {
     return undef;
     }
-    
-    
+        
     my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
     if ($srcdoc_exists) {
-    # do nothing more
+    # do nothing more - all the metadata has been extracted and associated with the srcdoc
     # no more need to access details of this $file => tidy up as you go
     delete $self->{'oai-files'}->{$file};
@@ -283 +250 @@
     }
 
-### print STDERR "**** !!!!! srcdoc_exists = $srcdoc_exists\n"; 
-    
     my $filename = $file;
     $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
@@ -304 +269 @@
     # include any metadata passed in from previous plugins 
     # note that this metadata is associated with the top level section
+    # this will include all the metadata from the oai file that we extracted
+    # during metadata_read
     $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
     
     # do plugin specific processing of doc_obj
-    print STDERR "raw xml 2 = $self->{'oai-files'}->{$file}->{'rawxml'}\n";
     my $text = $self->{'oai-files'}->{$file}->{'rawxml'};
     delete $self->{'oai-files'}->{$file};
@@ -331 +297 @@
 
 
-sub read_old {
-    my $self = shift (@_);  
-    
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-
-    my $filename = $file;
-    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
-
-    # block the srcdocs dir - we will process files in them when we find an OAI record for them
-    return 0 if ((-d $filename) && ($filename =~ m/srcdocs$/));
-    if ($self->SUPER::read(@_)) {
-    # Do encoding stuff
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
-    
-    my $url_array = $metadata->{'dc.Identifier'};
-    my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
-    
-    my $srcdoc_exists = 0;
-    my $srcdoc_pos = 0;
-    my $filename_dir = &util::filename_head($filename);
-    
-    for (my $i=0; $i<$num_urls; $i++) {
-        if ($url_array->[$i] !~ m/^(http|ftp):/) {
-        
-        my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
-        if (-e $src_filename) {
-            $srcdoc_pos = $i;
-            $srcdoc_exists = 1;
-        }
-        }
-    }
-    
-    if ($srcdoc_exists)
-    {
-        print $outhandle "OAIPlugin: passing metadata on to $url_array->[0]\n"
-        if ($self->{'verbosity'}>1);
-        
-        
-        # Make pretty print metadata table stick with src filename
-        my $ppmd_table = $self->{'ppmd_table'};
-        $metadata->{'prettymd'} = [ $ppmd_table ];
-        $self->{'ppmd_table'} = undef;
-        
-        return &plugin::read ($pluginfo, $filename_dir, $url_array->[0],
-                  $block_hash, $metadata, $processor, $maxdocs,
-                  $total_count, $gli);
-    }
-    else
-    {
-        # create a new document
-        my $doc_obj = new doc ($filename, "indexed_doc");
-        my $top_section = $doc_obj->get_top_section;
-        my $plugin_type = $self->{'plugin_type'};
-        
-        $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-        $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
-        $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
-        $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
-        $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
-        
-        # include any metadata passed in from previous plugins 
-        # note that this metadata is associated with the top level section
-        $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
-        
-        # do plugin specific processing of doc_obj
-        my $textref = \$self->{'rawxml'};
-        unless (defined ($self->process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
-        print STDERR "<ProcessingError n='$file'>\n" if ($gli);
-        return -1;
-        }
-        
-        # do any automatic metadata extraction
-        $self->auto_extract_metadata ($doc_obj);
-        
-        # add an OID
-        $self->add_OID($doc_obj);
-        
-        my $ppmd_table = $self->{'ppmd_table'};
-        $doc_obj->add_utf8_metadata($top_section,"prettymd",$ppmd_table);
-        $self->{'ppmd_table'} = undef;
-        
-        # process the document
-        $processor->process($doc_obj);
-        
-        $self->{'num_processed'} ++;
-        
-        return 1; # processed the file
-    }
-    }
-    else {
-    return undef;
-    }
-}
-
-
 # do plugin specific processing of doc_obj
 sub process {
@@ -450 +319 @@
     $$textref =~ s/\]/&#93;/g;
 
-##    print STDERR "*** adding text: $$textref\n";
-    
     $doc_obj->add_utf8_text($cursection, $$textref);
 
@@ -475 +342 @@
     my ($metaname, $metavalue_utf8) = @_;
 
-###    $metavalue_utf8 =~ s/hdl\.handle\.net/mcgonagall.cs.waikato.ac.nz:8080\/dspace\/handle/;
     $metavalue_utf8 = &util::hyperlink_text($metavalue_utf8);
 
@@ -596 +462 @@
     while ($inner_metadata_text =~ m/<([^ >]+).*?>(.*?)<\/\1>(.*)/s)
     {
-        # if URL given for document as identifier metadata, store it ...
-        # $doc_obj->add_utf8_metadata($cursection, "URL", $web_url);
 
         my $metaname = $1;
         my $metavalue = $2;
         $inner_metadata_text = $3;
-
-#       print STDERR "*** metaname = $metaname\n";
-#       print STDERR "*** metavalue = $metavalue\n";
 
         # $metaname =~ s/^(dc:)?(.)/\u$2/; # strip of optional prefix and uppercase first letter
@@ -611 +472 @@
         {
         $metaname = "$top_level_prefix.$metaname";
-#       print STDERR "*** metaname = $metaname\tmetavalue = $metavalue\n";
         }
         $metaname =~ s/\.(.)/\.\u$1/;
@@ -619 +479 @@
         $metavalue =~ s/\[/&#91;/g;
         $metavalue =~ s/\]/&#93;/g;
-
-
-#       if ($metaname eq "Identifier")
-#       {
-#       # name clashes with GSDL reserved metadata name for hash id
-#       $metaname = "URL";
-#       }
 
         if (defined $metadata->{$metaname})