Changeset 17318 for gsdl/trunk/bin

Timestamp:

2008-09-18T10:02:41+12:00 (16 years ago)

Author:

kjdon

Message:

tidied this up a bit. Modified the bit using metadata_read so that it works with my fixed OAIPlugin. Also allows multiple records to come back from metadata_read, but this hasn't been tested yet as its not currently used.

File:

• gsdl/trunk/bin/script/explode_metadata_database.pl (modified) (12 diffs)

gsdl/trunk/bin/script/explode_metadata_database.pl

@@ -227 +227 @@
 
     my @metadata_records = split(/$split_exp/, $text);
-    print STDERR "Number of records: " . scalar(@metadata_records) . "\n";
+    my $total_num_records = scalar(@metadata_records);
+    print STDERR "Number of records: $total_num_records\n";
     
     # Write the metadata from each record to the metadata.xml file
@@ -235 +236 @@
         
         # Check if we need to start a new directory for these records
-        check_need_new_directory($exploded_base_dir,$record_number,$records_per_folder,
-                     \@metadata_records,\$documents_directory);
+        check_need_new_directory($exploded_base_dir,$record_number,
+                     $records_per_folder,$total_num_records,
+                     \$documents_directory);
         # Use the plugin's process function to avoid duplicating code
         my $doc_obj = new doc($filename, "nonindexed_doc");
@@ -249 +251 @@
                 
         
-        check_close_directory($record_number,$records_per_folder,\@metadata_records);
+        check_close_directory($record_number,$records_per_folder,$total_num_records);
         
         $record_number = $record_number + 1;
@@ -255 +257 @@
     }
     else {
-    # Call metadata_read to sets up associated metadata 
+    # Call metadata_read to set up associated metadata 
 
     my $pluginfo = undef;
@@ -264 +266 @@
     my $gli = undef;
 
-    my $extrametakeys = {};
+    my $extrametakeys = [];
     my $extrametadata = {};
 
@@ -271 +273 @@
                 $extrametakeys, $extrametadata, $processor, $maxdocs, $gli);
 
-
-    my $documents_directory = need_new_directory($exploded_base_dir);
-
-    # Attach metadata to object
-    # => use the plugin's extra_metadata function to avoid duplicating code
-    my $doc_obj = new doc($filename, "nonindexed_doc");
-
-    $plugobj->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $extrametadata);
-
-    # Try to get a doc to attach the metadata to
-    # If no match found, create a dummy .nul file
-    attach_metadata_or_make_nul_doc($document_field, $doc_obj, undef, 
-                    $documents_directory, $orig_base_dir,
-                    $document_prefix, $document_suffix, $metadata_set, $verbosity);
-
-
-    close_directory();
-    }
-
+    my $total_num_records = scalar (@$extrametakeys);
+    print STDERR "Number of records: $total_num_records\n";
+    my $record_number = 1;
+    my $documents_directory;
+    foreach my $record (@$extrametakeys) {
+        &check_need_new_directory($exploded_base_dir, $record_number, $records_per_folder, $total_num_records, \$documents_directory);
+        
+        # Attach metadata to object
+        # => use the plugin's extra_metadata function to avoid duplicating code
+        my $doc_obj = new doc($filename, "nonindexed_doc");
+        # all the metadata has been extracted into extrametadata
+        $plugobj->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $extrametadata->{$record});
+
+        # Try to get a doc to attach the metadata to
+        # If no match found, create a dummy .nul file
+        attach_metadata_or_make_nul_doc($document_field, $doc_obj, $record_number, $documents_directory, $orig_base_dir, $document_prefix, $document_suffix, $metadata_set, $verbosity);
+        
+        &check_close_directory($record_number,$records_per_folder,$total_num_records);
+        
+        $record_number = $record_number + 1;
+
+    }
+    }
 
     # Explode means just that: the original file is deleted
@@ -326 +332 @@
 sub check_need_new_directory
 {
-    my ($exploded_base_dir,$record_number, $records_per_folder,$metadata_records,
-    $documents_dir_ref) = @_;
+    my ($exploded_base_dir,$record_number, $records_per_folder,
+    $total_num_records, $documents_dir_ref) = @_;
     
 
@@ -334 +340 @@
     my $documents_directory = $exploded_base_dir;
 
-    if (scalar(@$metadata_records) > $records_per_folder) {
+    if ($total_num_records > $records_per_folder) {
         $documents_directory .= "." . sprintf("%8.8d", $record_number);
     }
@@ -366 +372 @@
         my $document_file_full = $document_prefix . $value . $document_suffix;
 
+        # this either downloads/copies the document, or creates a nul file for it.
         $document_file = &obtain_document($document_file_full, $documents_directory, $orig_base_dir, $verbosity);
         &write_metadata_xml_file_entry(METADATA_XML_FILE, $document_file, $record_metadata, $metadata_set);
         }
     }
-    }
-    
-    # Create a dummy .nul file if we haven't obtained any documents for this record
+    }
+    
+    # Create a dummy .nul file if we haven't obtained a document (or null file) for this record
     if (not defined $document_file) {
 
@@ -399 +406 @@
 sub check_close_directory
 {
-    my ($record_number,$records_per_folder,$metadata_records) = @_;
-
-    if (($record_number % $records_per_folder) == 0 || $record_number == scalar(@$metadata_records)) {
+    my ($record_number,$records_per_folder,$total_num_records) = @_;
+
+    if (($record_number % $records_per_folder) == 0 || $record_number == $total_num_records) {
     # Finish and close the metadata.xml file
     close_directory();
@@ -511 +518 @@
     $local_document_file = &util::filename_cat($documents_directory, $document_file_name);
 
-    &util::cp($document_file_full, $documents_directory);
-
+    if (-e $document_file_full) {
+        &util::cp($document_file_full, $documents_directory);
+    }
+    
     # Check the document was obtained successfully
     if (!-e $local_document_file) {
@@ -531 +540 @@
     open(NULL_FILE, ">$local_document_file.nul");
     close(NULL_FILE);
+    print STDERR "Creating a nul document $document_file_name\n";
     }