Changeset 17216

Timestamp:

2008-09-08 14:59:50 (3 months ago)

Author:

kjdon

Message:

trying to get OAI files exploding. Have copied in some code from one of David's obsolete files. I think it works but haven't tested fully yet. Wanted to get the code committed though.

Files:

• gsdl/trunk/perllib/plugins/OAIPlugin.pm (modified) (2 diffs)

gsdl/trunk/perllib/plugins/OAIPlugin.pm

@@ -35 +35 @@
-use ReadXMLFile;
-use ReadTextFile; # needed for subroutine textcat_get_language_encoding
+use metadatautil;
-
-sub BEGIN {
@@ -160 +161 @@
-
-
+sub metadata_read {
+    my $self = shift (@_);  
+
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $extrametakeys, $extrametadata, $processor, $maxdocs, $gli) = @_;
+
+    # can we process this file??
+    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+    return undef unless $self->can_process_this_file($filename_full_path);
+    
+    my $total_count = 0; # is total count used?
+    if ($self->SUPER::read($pluginfo,$base_dir,$file,$block_hash,$metadata,$processor,$maxdocs,$total_count, $gli)) {
+        # calling "SUPER::read" at this point sets up $metadata
+        # data-structure.  We can then, later, in OAIPlug::read decide
+        # whether this $metadata will stick to an accompanying file,
+        # or else needs a new doc object to be formed that contains
+        # purely metadata
+                
+        $self->{'metadata'} = undef;
+        
+        #my $url_array = $metadata->{'gi.Sourcedoc'};
+        my $url_array = $metadata->{'dc.Identifier'};
+        my $num_urls = (defined $url_array) ? scalar(@$url_array) : 0;
+
+        my $srcdoc_exists = 0;
+        my $srcdoc_pos = 0;
+        my $filename_dir = &util::filename_head($filename_full_path);
+            
+        for (my $i=0; $i<$num_urls; $i++) {
+            
+            if ($url_array->[$i] !~ m/^(https?|ftp):/) {
+                
+                my $src_filename = &util::filename_cat($filename_dir, $url_array->[$i]);
+                
+                if (-e $src_filename) {
+                    $srcdoc_pos = $i;
+                    $srcdoc_exists = 1;
+                    last;
+                }
+            }
+        }
+        
+        
+        if ($srcdoc_exists)
+        {
+            $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 1;
+            
+###             print STDERR "**** storing OAI file: $file\n";
+            
+            # Make pretty print metadata table stick with src filename
+            my $ppmd_table = $self->{'ppmd_table'};
+            
+            $metadata->{'prettymd'} = [ $ppmd_table ];
+            $self->{'ppmd_table'} = undef;
+            
+        }
+        else {
+            $self->{'oai-files'}->{$file}->{'srcdoc_exists'} = 0;
+        }
+        
+    }
+    else {
+        return undef;
+    }
+}
-
-
-sub read {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+
+###    print STDERR "**** checking OAI read: $file\n";
+
+    if (defined $self->{'oai-files'}->{$file}) {
+        
+        my $srcdoc_exists = $self->{'oai-files'}->{$file}->{'srcdoc_exists'};
+
+        # no more need to access details of this $file => tidy up as you go
+        delete $self->{'oai-files'}->{$file};
+
+###     print STDERR "**** !!!!! srcdoc_exists = $srcdoc_exists\n";     
+        if (!$srcdoc_exists)
+        {
+
+            my $filename = $file;
+            $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+
+            # Do encoding stuff on metadata
+            my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+
+            # create a new document
+            my $doc_obj = new doc ($filename, "indexed_doc");
+            my $top_section = $doc_obj->get_top_section;
+            my $plugin_type = $self->{'plugin_type'};
+            
+            $doc_obj->add_utf8_metadata($top_section, "Language", $language);
+            $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+            $doc_obj->add_utf8_metadata($top_section, "Plugin", $plugin_type);
+            $doc_obj->add_metadata($top_section, "FileFormat", "OAI");
+            $doc_obj->add_metadata($top_section, "FileSize", (-s $filename));
+            
+            # include any metadata passed in from previous plugins 
+            # note that this metadata is associated with the top level section
+            $self->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $metadata);
+            
+            # do plugin specific processing of doc_obj
+            my $textref = \$self->{'rawxml'};
+            unless (defined ($self->process($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj))) {
+                print STDERR "<ProcessingError n='$file'>\n" if ($gli);
+                return -1;
+            }
+            
+            # do any automatic metadata extraction
+            $self->auto_extract_metadata ($doc_obj);
+            
+            # add an OID
+            $self->add_OID($doc_obj);
+            
+            my $prettymds = $self->{'prettymd'};
+            foreach my $prettymd (@$prettymds) {
+                $doc_obj->add_utf8_metadata($top_section,"prettymd",$prettymd);
+            }
+            $self->{'prettymd'} = undef;
+            
+            # process the document
+            $processor->process($doc_obj);
+            
+            $self->{'num_processed'} ++;
+            
+            return 1; # processed the file
+        }
+    }
+    else {
+        return undef;
+    }
+}
+
+
+sub read_old {
-    my $self = shift (@_);  
-