Changeset 26183

Timestamp:

2012-09-13T10:12:59+12:00 (12 years ago)

Author:

ak19

Message:

Committing changes Dr Bainbridge and I made to the g2f-import and g2f-building long ago. Note g2f-import runs both import and export.

Location:

main/trunk/greenstone2/bin/script

Files:

• g2f-buildcol.pl (modified) (4 diffs)
• g2f-import.pl (modified) (2 diffs)

main/trunk/greenstone2/bin/script/g2f-buildcol.pl

@@ -21 +21 @@
     $ENV{'FEDORA_PROTOCOL'} = "http" if (!defined $ENV{'FEDORA_PROTOCOL'});
     $ENV{'FEDORA_PID_NAMESPACE'} = "greenstone" if (!defined $ENV{'FEDORA_PID_NAMESPACE'});
+    $ENV{'FEDORA_PREFIX'} = "/fedora" if (!defined $ENV{'FEDORA_PREFIX'});
 
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/");
@@ -35 +36 @@
 use printusage;
 use parse2;
+use cfgread;
+use colcfg;
 
 use g2futil;
 
+use dbutil;
 
 my $arguments = 
@@ -213 +217 @@
     foreach my $hd (@hash_dirs) {
 
+        my $hash_id = &g2futil::get_hash_id($hd);
+        
+        if (defined $hash_id) {
+
+        my $pid = "$pid_namespace:$gs_col-$hash_id";
+        
+
+        my $dsinfo_status = &g2futil::run_datastore_info($pid,$options);
+        
+        if ($dsinfo_status == 0) {
+            print "  $pid being updated.\n";        
+            &g2futil::run_purge($pid,$options);
+        }
+        else {
+            print "  $pid not present.\n";
+        }
+        }
+
         my $docmets_filename 
         = &util::filename_cat($hd,"docmets.xml");
@@ -227 +249 @@
     else {
     print STDERR "Error: Unable to open directory $export_dir: $!\n";
-    exit;
-    }
-
-
+    exit 1;
+    }
+
+
+# can possibly use inexport instead of running buildcol.pl through system()
+    print STDERR "**** Just for now, also run Greenstone's buildcol.pl\n";
+
+    my $gs_opts = " -verbosity $verbosity";
+    $gs_opts .= " -gli" if ($gli);
+    $gs_opts .= " -collectdir \"$collectdir\"" if ($collectdir);
+    $gs_opts .= " -mode infodb";
+
+    my $gs_buildcol_arguments = "$gs_opts $gs_col";
+
+    &g2futil::run_cmd("buildcol.pl", $gs_buildcol_arguments, $options);
+
+    # read in collect cfg file to work out db type
+    my $collectcfg = &util::filename_cat ($collectdir, $gs_col, "etc", "collectionConfig.xml");
+    #print STDERR "**** collectcfg file: $collectcfg\n";
+    unless(open(FIN, "<$collectcfg")) { 
+    print STDERR "g2f-buildcol.pl: Unable to open $collectcfg...ERROR: $!\n";
+    exit 1; 
+    }
+    close(FIN);
+
+    # for now we assume GS3, since that's what the following gets implemented for
+    my $collect_cfg = &colcfg::read_collection_cfg ($collectcfg, "gs3");
+    # get the database type for this collection from its configuration file (may be undefined)
+    my $infodbtype = $collect_cfg->{'infodbtype'} || &dbutil::get_default_infodb_type();
+ 
+    # open .gdbm database file in building/text/$colname.gdb, using dbutil
+    my $colname = $gs_col;
+    $colname =~ s/(:?\\|\/)(.*)$/$1/; # remove any collect group from collection name to get tailname
+
+    my $building_txt_dir = &util::filename_cat ($collectdir, $gs_col, "building", "text");
+    my $building_txt_db = &dbutil::get_infodb_file_path($infodbtype, "$colname", $building_txt_dir);
+
+    # foreach key that matches http://dir1/dir2/....file.xxx
+    my $db_keys = {};
+    &dbutil::read_infodb_keys($infodbtype,$building_txt_db, $db_keys);
+
+    foreach my $key (keys %$db_keys) {
+    if($key =~ m@^http://@) {
+
+        # get value for the key
+        my $src_rec_string = &dbutil::read_infodb_entry($infodbtype,$building_txt_db, $key);
+        my $src_rec = &dbutil::convert_infodb_string_to_hash($src_rec_string);
+        my $OID_hash_value = $src_rec->{'section'}->[0];
+        $OID_hash_value = "$pid_namespace:$gs_col-".$OID_hash_value; # convert to fedoraPID
+
+        #   its fedora pid = "greenstone-http:$colname-http:||dir|file.xxx"
+        # except that fedorapids don't like extra colons and don't like |
+        my $fedora_identifier = "$pid_namespace-http:$gs_col-$key";
+        # CAN'T HAVE | OR : (as in "http:||one|two.html") in fedoraPID
+        $key =~ s@/@_@g; 
+        $key =~ s@:@-@g;
+        my $fedora_pid = "$pid_namespace-http:$gs_col-$key";
+
+        #   To run fedora ingest on the new file need to have sensible
+        #   filenames that won't offend windows     
+        my $fedora_key_file_name = "$fedora_pid";
+        $fedora_key_file_name =~ s@\.@-@g;
+        $fedora_key_file_name =~ s/\:/=/g;
+        $fedora_key_file_name .= ".xml";
+        print STDERR "+++++ fpid: $fedora_pid, fedora-key filename: $fedora_key_file_name\n";
+
+        #   write out a FedoraMets File for this key (in /tmp)
+        #   -> it has one metadata value, which is 'dc:title' = HASHxxxxxx
+        
+         # The HASHID shouldn't be the title: then will have
+         # duplicate titles and it will be hard to search for
+         # unique ones. What about making the filename the
+         # dc.title and the HASHID the dc.identifier
+
+        my $contents = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n";
+        $contents .= "<mets:mets xmlns:mets=\"http://www.loc.gov/METS/\"\n";
+        $contents .= " xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n";
+        $contents .= " xmlns:gsdl3=\"http://www.greenstone.org/namespace/gsdlmetadata/1.0/\"\n";
+        $contents .= " xmlns:xlink=\"http://www.w3.org/1999/xlink\"\n";
+        $contents .= " xsi:schemaLocation=\"http://www.loc.gov/METS/\n";
+        $contents .= " http://www.loc.gov/standards/mets/mets.xsd\n";
+        $contents .= " http://www.greenstone.org/namespace/gsdlmetadata/1.0/\n";
+        $contents .= " http://www.greenstone.org/namespace/gsdlmetadata/1.0/gsdl_metadata.xsd\"\n";
+        $contents .= " OBJID=\"$fedora_pid\"\n";
+#       $contents .= " OBJID=\"greenstone:$gs_col-HASH1f814d07252c354039ee11\"\n";
+        $contents .= " TYPE=\"FedoraObject\" LABEL=\"$fedora_pid\" EXT_VERSION=\"1.1\">\n";
+        $contents .= "<mets:metsHdr RECORDSTATUS=\"A\"/>\n";
+        $contents .= "   <mets:amdSec ID=\"DC\" >\n";
+        $contents .= "      <mets:techMD ID=\"DC.0\">\n";
+        $contents .= "         <mets:mdWrap LABEL=\"Metadata\" MDTYPE=\"OTHER\" OTHERMDTYPE=\"gsdl3\" ID=\"DCgsdl1\">\n";
+        $contents .= "            <mets:xmlData>\n";
+        $contents .= "               <oai_dc:dc xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" >\n";
+        $contents .= "                  <dc:title>$OID_hash_value</dc:title>\n";
+#       $contents .= "                  <dc:identifier>$fedora_identifier</dc:identifier>\n";
+        $contents .= "               </oai_dc:dc>\n";
+        $contents .= "            </mets:xmlData>\n";
+        $contents .= "         </mets:mdWrap>\n";
+        $contents .= "      </mets:techMD>\n";
+        $contents .= "   </mets:amdSec>\n";
+        $contents .= "</mets:mets>\n";      
+
+   
+        #   write out the file and then run fedora ingest on that file
+        #   The file gets purged in g2f-import.pl, so don't remove it from export dir now
+        my $fedora_key_file_path = &util::filename_cat($export_dir, $fedora_key_file_name);
+        unless(open(FOUT, ">$fedora_key_file_path")) { 
+        print STDERR "g2f-buildcol.pl: Unable to open $fedora_key_file_path...ERROR: $!\n";
+        exit 1; 
+        }
+        print FOUT $contents;
+        close(FOUT);
+
+        print STDERR "<Build>\n" if $gli;
+        print STDERR "Ingesting $fedora_key_file_name\n";
+        print STDERR "#### ".join(",", %$options)."\n";
+
+        &g2futil::run_ingest($fedora_key_file_path,$options);
+        print STDERR "</Build>\n" if $gli;
+    }
+    
+    }
+
+
+    # If successful!!! Then need to think about:
+    #    [CLX] nodes
+    #    Doing this with FedoraMETSPlugin
+
+    
 }
 

main/trunk/greenstone2/bin/script/g2f-import.pl

@@ -212 +212 @@
     # readdir
     if (opendir(DIR, $export_dir)) {
-
+        my @xml_files = grep { $_ =~ m/^greenstone-http.*\.xml$/ } readdir(DIR);
         closedir DIR;
+
+        # purge all the (URL,hashID) metadata files that we inserted
+        # into fedora at the end of g2f-buildcol.pl
+        # convert the filenames into fedora-pids
+        # filename = greenstone-http=tmpcol-http-__test1-html.xml -> fpid = greenstone-http:tmpcol-http-__test1.html
+        foreach my $file (@xml_files) {
+        my $fedora_pid = $file;
+        $fedora_pid =~ s/\.xml$//;
+        $fedora_pid =~ s/\=/:/;
+        $fedora_pid =~ s/(.*)-(.*)$/$1.$2/;
+        
+        print STDERR "#### fedora_pid: $fedora_pid\n";
+        &g2futil::run_purge($fedora_pid,$options); # displays error message if first time (nothing to purge)
+        }
+
         my @hash_dirs = &g2futil::get_all_hash_dirs($export_dir,$maxdocs);
 
@@ -246 +261 @@
     print "***\n";
 
-    my $gs_export_opts = "-saveas FedoraMETS -fedora_namespace $pid_namespace -verbosity $verbosity";
-
-    $gs_export_opts .= " -gli" if ($gli);
-
-    $gs_export_opts .= " -language $language" if ($language);
-    $gs_export_opts .= " -collectdir \"$collectdir\"" if ($collectdir);
-    $gs_export_opts .= " -removeold" if ($removeold);
-    $gs_export_opts .= " -maxdocs $maxdocs" if ($maxdocs);
-
-    $gs_export_opts .= " -exportdir \"$export_dir\"";
+    my $gs_export_opts = "-saveas FedoraMETS -fedora_namespace $pid_namespace";
+
+    my $gs_opts = " -verbosity $verbosity";
+    $gs_opts .= " -gli" if ($gli);
+
+    $gs_opts .= " -language $language" if ($language);
+    $gs_opts .= " -collectdir \"$collectdir\"" if ($collectdir);
+    $gs_opts .= " -removeold" if ($removeold);
+    $gs_opts .= " -maxdocs $maxdocs" if ($maxdocs);
+
+    $gs_export_opts .= " $gs_opts -exportdir \"$export_dir\"";
 
     my $gs_export_arguments = "$gs_export_opts $gs_col";
 
     &g2futil::run_cmd("export.pl", $gs_export_arguments, $options);
+
+    print STDERR "**** Just for now, also run Greenstone's import.pl\n";
+# if we have the FedoraMETSPlugIN then we wouldn't have to run import anymore
+    my $gs_import_arguments = "$gs_opts $gs_col";
+
+    &g2futil::run_cmd("import.pl", $gs_import_arguments, $options);
 }