Changeset 17216

Timestamp:

2008-09-08T14:59:50+12:00 (16 years ago)

Author:

kjdon

Message:

trying to get OAI files exploding. Have copied in some code from one of David's obsolete files. I think it works but haven't tested fully yet. Wanted to get the code committed though.

File:

• gsdl/trunk/perllib/plugins/OAIPlugin.pm (modified) (2 diffs)

gsdl/trunk/perllib/plugins/OAIPlugin.pm

@@ -35 +35 @@
 use ReadXMLFile;
 use ReadTextFile; # needed for subroutine textcat_get_language_encoding
+use metadatautil;
 
 sub BEGIN {
@@ -160 +161 @@
 
 
+sub metadata_read {
+    my $self = shift (@_);  
+
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file($filename_full_path);
+    
+    my $total_count = 0; # is total count used?
+    if ($self->SUPER::read($pluginfo,$base_dir,$file,$block_hash,$metadata,$processor,$maxdocs,$total_count, $gli)) {
+    # calling "SUPER::read" at this point sets up $metadata
+    # data-structure.  We can then, later, in OAIPlug::read decide
+    # whether this $metadata will stick to an accompanying file,
+    # or else needs a new doc object to be formed that contains
+    # purely metadata
+        
+    $self->{'metadata'} = undef;
+    
+    #my $url_array = $metadata->{'gi.Sourcedoc'};
+    my $url_array = $metadata->{'dc.Identifier'};
+    my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
+
+    my $srcdoc_exists = 0;
+    my $srcdoc_pos = 0;
+    my $filename_dir = &util::filename_head($filename_full_path);
+        
+    for (my $i=0; $i<$num_urls; $i++) {
+        
+        if ($url_array->[$i] !~ m/^(https?|ftp):/) {
+        
+        my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
+        
+        if (-e $src_filename) {
+            $srcdoc_pos = $i;
+            $srcdoc_exists = 1;
+            last;
+        }
+        }
+    }
+    
+    
+    if ($srcdoc_exists)
+    {
+        $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
+        
+###     print STDERR "**** storing OAI file: $file\n";
+        
+        # Make pretty print metadata table stick with src filename
+        my $ppmd_table = $self->{'ppmd_table'};
+        
+        $metadata->{'prettymd'} = [ $ppmd_table ];
+        $self->{'ppmd_table'} = undef;
+        
+    }
+    else {
+        $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 0;
+    }
+    
+    }
+    else {
+    return undef;
+    }
+}
 
 
 sub read {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+
+###    print STDERR "**** checking OAI read: $file\n";
+
+    if (defined $self->{'oai-files'}->{$file}) {
+    
+    my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
+
+    # no more need to access details of this $file => tidy up as you go
+    delete $self->{'oai-files'}->{$file};
+
+### print STDERR "**** !!!!! srcdoc_exists = $srcdoc_exists\n"; 
+    if (!$srcdoc_exists)
+    {
+
+        my $filename = $file;
+        $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+        # Do encoding stuff on metadata
+        my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+
+        # create a new document
+        my $doc_obj = new doc ($filename, "indexed_doc");
+        my $top_section = $doc_obj->get_top_section;
+        my $plugin_type = $self->{'plugin_type'};
+        
+        $doc_obj->add_utf8_metadata($top_section, "Language", $language);
+        $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+        $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
+        $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
+        $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
+        
+        # include any metadata passed in from previous plugins 
+        # note that this metadata is associated with the top level section
+        $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+        
+        # do plugin specific processing of doc_obj
+        my $textref = \$self->{'rawxml'};
+        unless (defined ($self->process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+        print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+        return -1;
+        }
+        
+        # do any automatic metadata extraction
+        $self->auto_extract_metadata ($doc_obj);
+        
+        # add an OID
+        $self->add_OID($doc_obj);
+        
+        my $prettymds = $self->{'prettymd'};
+        foreach my $prettymd (@$prettymds) {
+        $doc_obj->add_utf8_metadata($top_section,"prettymd",$prettymd);
+        }
+        $self->{'prettymd'} = undef;
+        
+        # process the document
+        $processor->process($doc_obj);
+        
+        $self->{'num_processed'} ++;
+        
+        return 1; # processed the file
+    }
+    }
+    else {
+    return undef;
+    }
+}
+
+
+sub read_old {
     my $self = shift (@_);