Changeset 8563

Timestamp:

2004-11-16T13:17:06+13:00 (19 years ago)

Author:

mdewsnip

Message:

Ripped all the obtaining referenced documents and exploding database code out into a new Perl script (bin/script/process_metadata_databases.pl) and changed the subfield character to be "^".

File:

• trunk/gsdl/perllib/plugins/ISISPlug.pm (modified) (10 diffs)

trunk/gsdl/perllib/plugins/ISISPlug.pm

@@ -65 +65 @@
     'type' => "string",
     'reqd' => "no",
-    'deft' => ", " },
-      { 'name' => "document_field",
-    'desc' => "{ISISPlug.document_field}",
-    'type' => "string",
-    'reqd' => "no",
-    'deft' => "" },
-      { 'name' => "document_prefix",
-    'desc' => "{ISISPlug.document_prefix}",
-    'type' => "string",
-    'reqd' => "no",
-    'deft' => "" },
-      { 'name' => "document_suffix",
-    'desc' => "{ISISPlug.document_suffix}",
-    'type' => "string",
-    'reqd' => "no",
-    'deft' => "" } 
+    'deft' => ", " }
       ];
 
@@ -108 +93 @@
 
 
-sub new {
+sub new
+{
     my $class = shift(@_);
 
@@ -115 +101 @@
              q^subfield_separator/.*/, ^, \$self->{'subfield_separator'},
              q^entry_separator/.*/<br>^, \$self->{'entry_separator'},
-             q^document_field/.*/^, \$self->{'document_field'},
-             q^document_prefix/.*/^, \$self->{'document_prefix'},
-             q^document_suffix/.*/^, \$self->{'document_suffix'},
              "allow_extra_options")) {
     print STDERR "\nIncorrect options passed to ISISPlug, check your collect.cfg configuration file\n";
@@ -132 +115 @@
 
 
-sub read
-{
-    my $self = shift(@_);
-    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
-
-    my $result = &SplitPlug::read($self, @_);
-    if ($file =~ /$self->{'process_exp'}/ && $self->{'document_field'}) {
-    &end_metadata_xml_file($self->{'documents_metadata_xml_file'});
-    }
-
-    return $result;
-}
-
-
-sub read_file {
+sub read_file
+{
     my $self = shift (@_);
     my ($filename, $encoding, $language, $textref) = @_;
 
-    my ($databasename) = ($filename =~ /([^\.]+)\.mst$/i);
+    my ($databasename) = ($filename =~ /(.*)\.mst$/i);
 
     # Check the associated .fdt and .xrf files exist
@@ -168 +138 @@
 
     my $reader = new multiread();
-    $reader->set_handle ('ISISPlug::FILE');
-    $reader->set_encoding ($encoding);
-    $reader->read_file ($textref);
+    $reader->set_handle('ISISPlug::FILE');
+    $reader->set_encoding($encoding);
+    $reader->read_file($textref);
 
     close(FILE);
@@ -182 +152 @@
     # Remove the line at the start so it is split and processed properly
     $$textref =~ s/^----------\n//;
-
-    # Obtain the documents specified in the CDS/ISIS database, if requested
-    if ($self->{'document_field'}) {
-    # Create a directory to store the document files
-    $self->{'documents_directory'} = $databasename . ".all";
-    if (-e $self->{'documents_directory'}) {
-        &util::rm_r($self->{'documents_directory'});
-    }
-    &util::mk_dir($self->{'documents_directory'});
-
-    # ...and a metadata.xml file for the document metadata (extracted from the database)
-    $self->{'documents_metadata_xml_file'} = &util::filename_cat($self->{'documents_directory'}, "metadata.xml");
-    if (-e $self->{'documents_metadata_xml_file'}) {
-        &util::rm($self->{'documents_metadata_xml_file'});
-    }
-    &begin_metadata_xml_file($self->{'documents_metadata_xml_file'});
-    }
 }
 
@@ -244 +197 @@
         my $subfieldname = "";
         if ($rawtagvalue =~ s/^\^([a-z])//) {
-            $subfieldname = "." . $1;
+            $subfieldname = "^$1";
         }
 
@@ -251 +204 @@
         my $metadatafieldname = $tagname . $subfieldname;
         my $metadatafieldvalue = $1;
-        # print "Metadata: $metadatafieldname -> $metadatafieldvalue\n";
 
         # Handle Keywords specially
@@ -278 +230 @@
     }
 
-    # print "Metadata: $tagname.all -> $completetagvalue\n";
-    $doc_obj->add_utf8_metadata($cursection, $tagname . ".all", $completetagvalue); 
-    }
-    # print "\n";
-    # Add fileFormat as the metadata
-    $doc_obj->add_metadata($cursection, "FileFormat", "CDS/ISIS");
+    $doc_obj->add_utf8_metadata($cursection, $tagname . "^all", $completetagvalue); 
+    }
+
     # Add the full record as the document text
     $$textref =~ s/\</&lt;/g;
     $$textref =~ s/\>/&gt;/g;
-    $doc_obj->add_utf8_text ($cursection, $$textref);
-
-    # Obtain the documents specified in the CDS/ISIS database, if requested
-    if ($self->{'document_field'}) {
-    my $document_field = $self->{'document_field'};
-    my $document_prefix = $self->{'document_prefix'} || "";
-    my $document_suffix = $self->{'document_suffix'} || "";
-
-    my $documents_directory = $self->{'documents_directory'};
-    my $document_obtained = 0;
-
-    # Look at all the metadata assigned to this record
-    my $record_metadata = $doc_obj->get_all_metadata($cursection);
-    foreach my $pair (@$record_metadata) {
-        my ($field, $value) = (@$pair);
-
-        # Does this metadata element specify a document to obtain?
-        if ($field eq $document_field) {
-        my $document_file_full = $document_prefix . $value . $document_suffix;
-
-        my $document_file = &obtain_document($self, $document_file_full, $documents_directory);
-        if ($document_file) {
-            $document_obtained = 1;
-            &write_metadata_xml_file($self->{'documents_metadata_xml_file'},
-                         $document_file, $record_metadata);
-        }
-        }
-    }
-
-    # If there was a document obtained for this record we don't want the record as well
-    if ($document_obtained) {
-        return 0;
-    }
-    }
+    $doc_obj->add_utf8_text($cursection, $$textref);
+
+    # Add FileFormat metadata
+    $doc_obj->add_utf8_metadata($cursection, "FileFormat", "CDS/ISIS");
 
     # Record was processed successfully (and there was no document obtained)
@@ -364 +283 @@
 
 
-sub obtain_document
-{
-    my $self = shift(@_);
-    my $document_file_full = shift(@_);
-    my $documents_directory = shift(@_);
-
-    my $outhandle = $self->{'outhandle'};
-    print $outhandle "Obtaining document file $document_file_full...\n"
-    if ($self->{'verbosity'} > 1);
-
-    my $document_file_name;
-    my $local_document_file;
-
-    # Document specified is on the web
-    if ($document_file_full =~ /^http:/ || $document_file_full =~ /^ftp:/) {
-    $document_file_full =~ /([^\/]+)$/;
-    $document_file_name = $1;
-    $local_document_file = &util::filename_cat($documents_directory, $document_file_name);
-
-    my $wget_options = "--quiet";
-    $wget_options = "--verbose" if ($self->{'verbosity'} > 2);
-    $wget_options .= " --timestamping";  # Only re-download files if they're newer
-    `wget $wget_options $document_file_full --output-document $local_document_file`;
-    }
-    # Document specified is on the disk
-    else {
-    my $dir_sep = &util::get_os_dirsep();
-    $document_file_full =~ /(.+$dir_sep)?(.*)$/;
-    $document_file_name = $2;
-    $local_document_file = &util::filename_cat($documents_directory, $document_file_name);
-
-    &util::cp($document_file_full, $documents_directory);
-    }
-
-    # Check the document was obtained successfully
-    if (!-e $local_document_file) {
-    print STDERR "WARNING: Could not obtain document file $document_file_full\n";
-    return undef;
-    }
-
-    return $document_file_name;
-}
-
-
-sub begin_metadata_xml_file
-{
-    my $metadata_xml_file = shift(@_);
-
-    open(METADATA_XML_FILE, ">$metadata_xml_file");
-    print METADATA_XML_FILE
-    "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n" .
-    "<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">\n" .
-    "<DirectoryMetadata>\n";
-    close(METADATA_XML_FILE);
-}
-
-
-sub write_metadata_xml_file
-{
-    my $metadata_xml_file = shift(@_);
-    my $file_name = shift(@_);
-    my $record_metadata = shift(@_);
-
-    # Make $file_name XML-safe
-    $file_name =~ s/</&lt;/g;
-    $file_name =~ s/>/&gt;/g;
-
-    open(METADATA_XML_FILE, ">>$metadata_xml_file");
-
-    print METADATA_XML_FILE
-    "\n" .
-        "  <FileSet>\n" .
-    "    <FileName>$file_name</FileName>\n" .
-    "    <Description>\n";
-
-    foreach my $pair (@$record_metadata) {
-    my ($field, $value) = (@$pair);
-
-    # We're only interested in metadata from the database
-    next if ($field eq "gsdlsourcefilename");
-    next if ($field eq "gsdldoctype");
-    next if ($field eq "Language");
-    next if ($field eq "Encoding");
-    next if ($field eq "Identifier");
-    next if ($field eq "Source");
-    next if ($field eq "SourceSegment");
-    next if ($field eq "Plugin");
-
-    # Make $value XML-safe
-    $value =~ s/</&lt;/g;
-    $value =~ s/>/&gt;/g;
-    
-    print METADATA_XML_FILE "      <Metadata name=\"$field\">$value</Metadata>\n";
-    }
-
-    print METADATA_XML_FILE
-    "    </Description>\n" .
-        "  </FileSet>\n";
-
-    close(METADATA_XML_FILE);
-}
-
-
-sub end_metadata_xml_file
-{
-    my $metadata_xml_file = shift(@_);
-
-    open(METADATA_XML_FILE, ">>$metadata_xml_file");
-    print METADATA_XML_FILE "\n</DirectoryMetadata>\n";
-    close(METADATA_XML_FILE);
-}
-
-
 1;