Changeset 12706

Timestamp:

2006-09-07T16:38:12+12:00 (18 years ago)

Author:

mdewsnip

Message:

Added a "-records_per_folder" option to explode_metadata_database.pl, which explodes the records into multiple folders.

Location:

trunk/gsdl

Files:

• bin/script/explode_metadata_database.pl (modified) (5 diffs)
• perllib/strings.properties (modified) (1 diff)

trunk/gsdl/bin/script/explode_metadata_database.pl

@@ -71 +71 @@
     'type' => "string",
     'reqd' => "no"},
+      { 'name' => "records_per_folder",
+    'desc' => "{explode.records_per_folder}",
+    'type' => "int",
+    'range' => "0,",
+    'deft' => "100",
+    'reqd' => "no" },
       { 'name' => "verbosity",
     'desc' => "{import.verbosity}",
@@ -93 +99 @@
 {
     my ($language, $input_encoding, $metadata_set, $plugin, 
-    $document_field, $document_prefix, $document_suffix, $verbosity);
+    $document_field, $document_prefix, $document_suffix, $records_per_folder, $verbosity);
 
     my $xml = 0;
@@ -174 +180 @@
     # Use the plugin's read_file function to avoid duplicating code
     $plugobj->read_file($filename, $input_encoding, undef, \$text);
-
     # is there any text in the file??
     die "\n" unless length($text);
+
     # Create a directory to store the document files...
-    my ($documents_directory) = ($filename =~ /(.*)\.[^\.]+$/);
-    if (-d $documents_directory) {
-    die "Error: document directory $documents_directory already exists (bailing).\n";
-    }
-    &util::mk_dir($documents_directory);
-
-    # ...and a metadata.xml file for the document metadata (extracted from the database)
-    my $documents_metadata_xml_file = &util::filename_cat($documents_directory, "metadata.xml");
-    if (-e $documents_metadata_xml_file) {
-    die "Error: document metadata.xml file $documents_metadata_xml_file already exists (bailing).\n";
-    }
-
-    # Start the metadata.xml file
-    open(METADATA_XML_FILE, ">$documents_metadata_xml_file");
-    print METADATA_XML_FILE
-    "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n" .
-    "<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">\n" .
-    "<DirectoryMetadata>\n";
+    my ($documents_directory_base) = ($filename =~ /(.*)\.[^\.]+$/);
 
     # Split the text into records, using the plugin's split_exp
     my $split_exp = $plugobj->{'split_exp'};
     my @metadata_records = split(/$split_exp/, $text);
-    print STDERR "Number of records: " . @metadata_records . "\n";
+    print STDERR "Number of records: " . scalar(@metadata_records) . "\n";
 
     # Write the metadata from each record to the metadata.xml file
-    my $record_number = 0;
+    my $record_number = 1;
+    my $documents_directory;
     foreach my $record_text (@metadata_records) {
-    
+    # Check if we need to start a new directory for these records
+    if (($record_number % $records_per_folder) == 1) {
+        $documents_directory = $documents_directory_base;
+        if (scalar(@metadata_records) > $records_per_folder) {
+        $documents_directory .= "." . sprintf("%8.8d", $record_number);
+        }
+        if (-d $documents_directory) {
+        die "Error: document directory $documents_directory already exists (bailing).\n";
+        }
+        &util::mk_dir($documents_directory);
+
+        my $documents_metadata_xml_file = &util::filename_cat($documents_directory, "metadata.xml");
+        if (-e $documents_metadata_xml_file) {
+        die "Error: documents metadata.xml file $documents_metadata_xml_file already exists (bailing).\n";
+        }
+
+        # Start the metadata.xml file
+        open(METADATA_XML_FILE, ">$documents_metadata_xml_file");
+        print METADATA_XML_FILE
+        "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n" .
+        "<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">\n" .
+        "<DirectoryMetadata>\n";
+    }
+
     # Use the plugin's process function to avoid duplicating code
     my $doc_obj = new doc($filename, "nonindexed_doc");
@@ -227 +240 @@
     # do we need to create a dummy doc??
     if (not defined $document_file) {
-        $record_number = $record_number + 1;
         $document_file = sprintf("%8.8d", $record_number) . ".nul";
-
         open(DUMMY_FILE, ">$documents_directory/$document_file");
         close(DUMMY_FILE);
@@ -235 +246 @@
 
     &write_metadata_xml_file_entry(METADATA_XML_FILE, $document_file, $record_metadata, $metadata_set);
-    }
-
-    # Finish and close the metadata.xml file
-    print METADATA_XML_FILE "\n</DirectoryMetadata>\n";
-    close(METADATA_XML_FILE);
+
+    if (($record_number % $records_per_folder) == 0 || $record_number == scalar(@metadata_records)) {
+        # Finish and close the metadata.xml file
+        print METADATA_XML_FILE "\n</DirectoryMetadata>\n";
+        close(METADATA_XML_FILE);
+    }
+    $record_number = $record_number + 1;
+    }
 
     # Explode means just that: the original file is deleted

trunk/gsdl/perllib/strings.properties

@@ -177 +177 @@
 explode.plugin: Plugin to use for exploding
 explode.params: [options] filename
+explode.records_per_folder: The number of records to put in each subfolder.
+
 # -- exportcol.pl --