Changeset 9147

Timestamp:

2005-02-23T14:01:04+13:00 (19 years ago)

Author:

kjdon

Message:

added in lots more options, mainly to do with downloading files

File:

• trunk/gsdl/bin/script/explode_metadata_database.pl (modified) (7 diffs)

trunk/gsdl/bin/script/explode_metadata_database.pl

@@ -25 +25 @@
     'desc' => "{explode.encoding}",
     'type' => "enum",
-    'deft' => "iso_8859_1",
+    'deft' => "auto",
     'list' => $unicode_list,
-    'reqd' => "no" } ,
+    'reqd' => "no" },
       { 'name' => "metadata_set",
     'desc' => "{explode.metadata_set}",
     'type' => "string",
-    'reqd' => "no" } ,
+    'reqd' => "no",
+    'hiddengli' => "yes"},
       { 'name' => "plugin",
     'desc' => "{explode.plugin}",
     'type' => "string",
-    'reqd' => "yes" },
+    'reqd' => "yes",
+    'hiddengli' => "yes"},
+      { 'name' => "document_field",
+    'desc' => "{explode.document_field}",
+    'type' => "string",
+    'reqd' => "no"},
+       { 'name' => "document_prefix",
+    'desc' => "{explode.document_prefix}",
+    'type' => "string",
+    'reqd' => "no"},
+      { 'name' => "document_suffix",
+    'desc' => "{explode.document_suffix}",
+    'type' => "string",
+    'reqd' => "no"},
       { 'name' => "filename_field",
     'desc' => "{explode.filename_field}",
     'type' => "string",
-    'reqd' => "no"}
+    'reqd' => "no"}     
       ];
     
@@ -49 +63 @@
 sub main
 {
-    my ($encoding, $metadata_set, $plugin, $filename_field);
-
-    
+    my ($encoding, $metadata_set, $plugin, $filename_field, 
+    $document_field, $document_prefix, $document_suffix);
+
+    my $xml = 0;
     # Parse command line arguments
     if (!parsargv::parse(\@ARGV, 
+             'language/.*/', \$language,
              'input_encoding/.*/auto', \$encoding,
              'metadata_set/.*/', \$metadata_set,
              'plugin/.*/', \$plugin,
-             'filename_field/.*/', \$filename_field)) {
+             'filename_field/.*/', \$filename_field,
+             'document_field/.*/', \$document_field,
+             'document_prefix/.*/', \$document_prefix,
+             'document_suffix/.*/', \$document_suffix,
+             q^xml^, \$xml)) {
     &PrintUsage::print_txt_usage($options, "{explode.params}");
     die "\n";
+    }
+    
+    # If $language has been specified, load the appropriate resource bundle
+    # (Otherwise, the default resource bundle will be loaded automatically)
+    if ($language) {
+    &gsprintf::load_language_specific_resource_bundle($language);
+    }
+
+    if ($xml) {
+        &PrintUsage::print_xml_usage($options);
+    print "\n";
+    return;
     }
 
@@ -89 +121 @@
 
     #check filename field
-    
+    if (defined $fileanme_field && $filenmae_field eq "") {
+    undef $filename_field;
+    }
     my $plugobj;
     require "$plugin.pm";
@@ -137 +171 @@
     my $doc_obj = new doc($filename, "nonindexed_doc");
     $plugobj->process(\$record_text, undef, undef, $filename, undef, $doc_obj, 0);
-    # try to get a file name
+    # Get all the metadata assigned to this record
+    my $record_metadata = $doc_obj->get_all_metadata($doc_obj->get_top_section());
     my $document_file;
-    if (defined $filename_field) {
-        my $meta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $filename_field);
-        if (defined $meta) {
-        $meta =~ s/&\w{1,10};//g; # remove entities
-        $document_file = "$meta.nul";
-        my $num = 0;
-        while (-e "$documents_directory/$document_file") {
-            $num++;
-            $document_file = "$meta$num.nul";
+    
+    # try to get a doc to attach the metadata to
+    if (defined $document_field) {
+        foreach my $pair (@$record_metadata) {
+        my ($field, $value) = (@$pair);
+        
+        # Does this metadata element specify a document to obtain?
+        if ($field eq $document_field) {
+            my $document_file_full = $document_prefix . $value . $document_suffix;
+            $document_file = &obtain_document($self, $document_file_full, $documents_directory);
+        }
+        }
+    }
+    # do we need to create a dummy doc??
+    if (not defined $document_file) {
+        # try to get a file name
+        if (defined $filename_field) {
+        my $meta = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $filename_field);
+        if (defined $meta) {
+            $meta =~ s/&\w{1,10};//g; # remove entities
+            $document_file = "$meta.nul";
+            my $num = 0;
+            while (-e "$documents_directory/$document_file") {
+            $num++;
+            $document_file = "$meta$num.nul";
+            }
+        } else {
+            $record_number = $record_number + 1;
+            $document_file = sprintf("%4.4d", $record_number) . ".nul";
         }
         } else {
@@ -153 +208 @@
         $document_file = sprintf("%4.4d", $record_number) . ".nul";
         }
-    } else {
-        $record_number = $record_number + 1;
-        $document_file = sprintf("%4.4d", $record_number) . ".nul";
+        open(DUMMY_FILE, ">$documents_directory/$document_file");
+        close(DUMMY_FILE);
+
     }
     
-    open(DUMMY_FILE, ">$documents_directory/$document_file");
-    close(DUMMY_FILE);
-
-    # Look at all the metadata assigned to this record
-    my $record_metadata = $doc_obj->get_all_metadata($cursection);
+
     &write_metadata_xml_file_entry(METADATA_XML_FILE, $document_file, $record_metadata, $metadata_set);
     }
@@ -171 +222 @@
 
     # Explode means just that: the original file is deleted
-    #&util::rm($filename);
+    &util::rm($filename);
 }
 
@@ -231 +282 @@
 }
 
+sub obtain_document
+{
+    my $self = shift(@_);
+    my $document_file_full = shift(@_);
+    my $documents_directory = shift(@_);
+
+    my $outhandle = $self->{'outhandle'};
+    print $outhandle "Obtaining document file $document_file_full...\n"
+    if ($self->{'verbosity'} > 1);
+
+    my $document_file_name;
+    my $local_document_file;
+
+    # Document specified is on the web
+    if ($document_file_full =~ /^http:/ || $document_file_full =~ /^ftp:/) {
+    $document_file_full =~ /([^\/]+)$/;
+    $document_file_name = $1;
+    $local_document_file = &util::filename_cat($documents_directory, $document_file_name);
+
+    my $wget_options = "--quiet";
+    $wget_options = "--verbose" if ($self->{'verbosity'} > 2);
+    $wget_options .= " --timestamping";  # Only re-download files if they're newer
+    `wget $wget_options $document_file_full --output-document $local_document_file`;
+    }
+    # Document specified is on the disk
+    else {
+    my $dir_sep = &util::get_os_dirsep();
+    $document_file_full =~ /(.+$dir_sep)?(.*)$/;
+    $document_file_name = $2;
+    $local_document_file = &util::filename_cat($documents_directory, $document_file_name);
+
+    &util::cp($document_file_full, $documents_directory);
+    }
+
+    # Check the document was obtained successfully
+    if (!-e $local_document_file) {
+    print STDERR "WARNING: Could not obtain document file $document_file_full\n";
+    return undef;
+    }
+
+    return $document_file_name;
+}
 
 &main(@ARGV);