Index: gsdl/trunk/bin/script/explode_metadata_database.pl
===================================================================
--- gsdl/trunk/bin/script/explode_metadata_database.pl	(revision 17313)
+++ gsdl/trunk/bin/script/explode_metadata_database.pl	(revision 17318)
@@ -227,5 +227,6 @@
 
 	my @metadata_records = split(/$split_exp/, $text);
-	print STDERR "Number of records: " . scalar(@metadata_records) . "\n";
+	my $total_num_records = scalar(@metadata_records);
+	print STDERR "Number of records: $total_num_records\n";
 	
 	# Write the metadata from each record to the metadata.xml file
@@ -235,6 +236,7 @@
 	    
 	    # Check if we need to start a new directory for these records
-	    check_need_new_directory($exploded_base_dir,$record_number,$records_per_folder,
-				     \@metadata_records,\$documents_directory);
+	    check_need_new_directory($exploded_base_dir,$record_number,
+				     $records_per_folder,$total_num_records,
+				     \$documents_directory);
 	    # Use the plugin's process function to avoid duplicating code
 	    my $doc_obj = new doc($filename, "nonindexed_doc");
@@ -249,5 +251,5 @@
 	    	    
 	    
-	    check_close_directory($record_number,$records_per_folder,\@metadata_records);
+	    check_close_directory($record_number,$records_per_folder,$total_num_records);
 	    
 	    $record_number = $record_number + 1;
@@ -255,5 +257,5 @@
     }
     else {
-	# Call metadata_read to sets up associated metadata 
+	# Call metadata_read to set up associated metadata 
 
 	my $pluginfo = undef;
@@ -264,5 +266,5 @@
 	my $gli = undef;
 
-	my $extrametakeys = {};
+	my $extrametakeys = [];
 	my $extrametadata = {};
 
@@ -271,23 +273,27 @@
 				$extrametakeys, $extrametadata, $processor, $maxdocs, $gli);
 
-
-	my $documents_directory = need_new_directory($exploded_base_dir);
-
-	# Attach metadata to object
-	# => use the plugin's extra_metadata function to avoid duplicating code
-	my $doc_obj = new doc($filename, "nonindexed_doc");
-
-	$plugobj->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $extrametadata);
-
-	# Try to get a doc to attach the metadata to
-	# If no match found, create a dummy .nul file
-	attach_metadata_or_make_nul_doc($document_field, $doc_obj, undef, 
-					$documents_directory, $orig_base_dir,
-					$document_prefix, $document_suffix, $metadata_set, $verbosity);
-
-
-	close_directory();
-    }
-
+	my $total_num_records = scalar (@$extrametakeys);
+	print STDERR "Number of records: $total_num_records\n";
+	my $record_number = 1;
+	my $documents_directory;
+	foreach my $record (@$extrametakeys) {
+	    &check_need_new_directory($exploded_base_dir, $record_number, $records_per_folder, $total_num_records, \$documents_directory);
+	    
+	    # Attach metadata to object
+	    # => use the plugin's extra_metadata function to avoid duplicating code
+	    my $doc_obj = new doc($filename, "nonindexed_doc");
+	    # all the metadata has been extracted into extrametadata
+	    $plugobj->extra_metadata ($doc_obj, $doc_obj->get_top_section(), $extrametadata->{$record});
+
+	    # Try to get a doc to attach the metadata to
+	    # If no match found, create a dummy .nul file
+	    attach_metadata_or_make_nul_doc($document_field, $doc_obj, $record_number, $documents_directory, $orig_base_dir, $document_prefix, $document_suffix, $metadata_set, $verbosity);
+	    
+	    &check_close_directory($record_number,$records_per_folder,$total_num_records);
+	    
+	    $record_number = $record_number + 1;
+
+	}
+    }
 
     # Explode means just that: the original file is deleted
@@ -326,6 +332,6 @@
 sub check_need_new_directory
 {
-    my ($exploded_base_dir,$record_number, $records_per_folder,$metadata_records,
-	$documents_dir_ref) = @_;
+    my ($exploded_base_dir,$record_number, $records_per_folder,
+	$total_num_records, $documents_dir_ref) = @_;
     
 
@@ -334,5 +340,5 @@
 	my $documents_directory = $exploded_base_dir;
 
-	if (scalar(@$metadata_records) > $records_per_folder) {
+	if ($total_num_records > $records_per_folder) {
 	    $documents_directory .= "." . sprintf("%8.8d", $record_number);
 	}
@@ -366,11 +372,12 @@
 		my $document_file_full = $document_prefix . $value . $document_suffix;
 
+		# this either downloads/copies the document, or creates a nul file for it.
 		$document_file = &obtain_document($document_file_full, $documents_directory, $orig_base_dir, $verbosity);
 		&write_metadata_xml_file_entry(METADATA_XML_FILE, $document_file, $record_metadata, $metadata_set);
 	    }
 	}
-	}
-    
-    # Create a dummy .nul file if we haven't obtained any documents for this record
+    }
+    
+    # Create a dummy .nul file if we haven't obtained a document (or null file) for this record
     if (not defined $document_file) {
 
@@ -399,7 +406,7 @@
 sub check_close_directory
 {
-    my ($record_number,$records_per_folder,$metadata_records) = @_;
-
-    if (($record_number % $records_per_folder) == 0 || $record_number == scalar(@$metadata_records)) {
+    my ($record_number,$records_per_folder,$total_num_records) = @_;
+
+    if (($record_number % $records_per_folder) == 0 || $record_number == $total_num_records) {
 	# Finish and close the metadata.xml file
 	close_directory();
@@ -511,6 +518,8 @@
 	$local_document_file = &util::filename_cat($documents_directory, $document_file_name);
 
-	&util::cp($document_file_full, $documents_directory);
-
+	if (-e $document_file_full) {
+	    &util::cp($document_file_full, $documents_directory);
+	}
+	
 	# Check the document was obtained successfully
 	if (!-e $local_document_file) {
@@ -531,4 +540,5 @@
 	open(NULL_FILE, ">$local_document_file.nul");
 	close(NULL_FILE);
+	print STDERR "Creating a nul document $document_file_name\n";
     }
 
