Changeset 20616 for gsdl/trunk

Timestamp:

2009-09-16T15:55:57+12:00 (15 years ago)

Author:

kjdon

Message:

first step at bring export into line with import regarding incremental export

File:

• gsdl/trunk/bin/script/export.pl (modified) (21 diffs)

gsdl/trunk/bin/script/export.pl

@@ -27 +27 @@
 
 
-# This program will export a particular collection into a specific Format (e.g. METS or DSpace)
+# This program will export a particular collection into a specific Format (e.g. METS or DSpace) by importing then saving as a different format.
 
 package export;
@@ -36 +36 @@
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/perl-5.8");
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts");
@@ -61 +62 @@
 use plugout;
 use manifest;
+use inexport;
 use util;
 use scriptutil;
@@ -143 +145 @@
     'reqd' => "no",
         'hiddengli' => "yes" },
-      { 'name' => "listall",
-    'desc' => "{export.listall}",
-    'type' => "flag",
-    'reqd' => "no" },
       { 'name' => "debug",
     'desc' => "{export.debug}",
@@ -159 +157 @@
         'modegli' => "3" },
       # does this make sense?
-#      { 'name' => "incremental",
-#   'desc' => "{import.incremental}",
-#   'type' => "flag",
-#   'hiddengli' => "yes" },
+      { 'name' => "incremental",
+    'desc' => "{import.incremental}",
+    'type' => "flag",
+    'hiddengli' => "yes" },
       { 'name' => "keepold",
     'desc' => "{export.keepold}",
@@ -172 +170 @@
     'type' => "flag",
     'reqd' => "no",
-    'modegli' => "3" },
+    'hiddengli' => "yes" },
       { 'name' => "language",
     'desc' => "{scripts.language}",
     'type' => "string",
     'reqd' => "no",
-    'modegli' => "3" },
+    'hiddengli' => "yes" },
       { 'name' => "maxdocs",
     'desc' => "{export.maxdocs}",
@@ -254 +252 @@
     'reqd' => "no",
     'hiddengli' => "yes" },
+      { 'name' => "listall",
+    'desc' => "{export.listall}",
+    'type' => "flag",
+    'reqd' => "no" },
       { 'name' => "xml",
     'desc' => "{scripts.xml}",
@@ -280 +282 @@
     $OIDtype, $OIDmetadata,
     $maxdocs, $statsfile, 
+    $gzip,
     $out, $faillog, $gli, $listall, 
     # plugout specific ones
@@ -289 +292 @@
     # other vars
     my ($configfilename, $collection, $collectcfg,
-    $expinfo_doc_filename, $export_info, 
+    $expinfo_doc_filename, $expinfo_src_filename, $export_info, 
     $gs_mode, 
     $processor, $pluginfo);
@@ -311 +314 @@
     }
 
-    
-    # these are options used by other things - we just set default values
-    # undef means will be set from config file if there
-    my $gzip = undef; 
-
+   
     # If $language has been specified, load the appropriate resource bundle
     # (Otherwise, the default resource bundle will be loaded automatically)
@@ -420 +419 @@
     }
     
+    if (defined $collectcfg->{'manifest'} && $manifest eq "") {
+    $manifest = $collectcfg->{'manifest'};
+    }
     if (defined $collectcfg->{'gzip'} && !$gzip) {
     if ($collectcfg->{'gzip'} =~ /^true$/i) {
@@ -433 +435 @@
     }
     
+    # groupsize is in import - does it make sense here??
+
     if (!defined $OIDtype || ($OIDtype !~ /^(hash|incremental|assigned|dirname)$/)) {
     if (defined $collectcfg->{'OIDtype'} && $collectcfg->{'OIDtype'} =~ /^(hash|incremental|assigned|dirname)$/) {
@@ -441 +445 @@
     }
 
+    if ((!defined $OIDmetadata) || ($OIDmetadata eq "")) {
+    if (defined $collectcfg->{'OIDmetadata'}) {
+        $OIDmetadata = $collectcfg->{'OIDmetadata'};
+    } else {
+        $OIDmetadata = "dc.Identifier"; # the default
+    }
+    }
+
     if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
     $debug = 1;
@@ -496 +508 @@
     } elsif ($saveas =~ m/^.*METS$/ || $saveas eq "MARCXML" ) {
 ##  $expinfo_doc_filename = &util::filename_cat ($exportdir, "export.inf");
-    my $doc_db = "archiveinf-doc";
-    $expinfo_doc_filename = &util::filename_cat ($exportdir, $doc_db);
+    $expinfo_doc_filename = &util::filename_cat ($exportdir,"archiveinf-doc" );
     &util::rename_gdbm_file($expinfo_doc_filename); # ensures gdb in case we have an existing legacy ldb one - can this happen?
     $expinfo_doc_filename .= ".gdb";
+    
+    $expinfo_src_filename = &util::filename_cat ($exportdir,"archiveinf-src" );
+    &util::rename_gdbm_file($expinfo_src_filename); # ensures gdb in case we have an existing legacy ldb one - can this happen?
+    $expinfo_src_filename .= ".gdb";
 
     }
@@ -506 +521 @@
     $export_info -> load_info ($expinfo_doc_filename);  
         
+    if ($manifest eq "") {
+    # Load in list of files in export folder from last export (if present)
+    $export_info->load_prev_import_filelist ($expinfo_src_filename);
+    }
+    
     my ($plugout); 
     if (defined $collectcfg->{'plugout'} && $collectcfg->{'plugout'} =~ /^(.*METS|DSpace|MARCXML)Plugout/) {
@@ -511 +531 @@
     }
     else{
-    if ($saveas !~ /^(.*METS|DSpace|MARCXML)$/) {
+    if ($saveas !~ /^(GreenstoneMETS|FedoraMETS|DSpace|MARCXML)$/) {
         push @$plugout,"GreenstoneMETSPlugout";
     }
@@ -524 +544 @@
     push @$plugout,("-verbosity",$verbosity) if (defined $verbosity);
     push @$plugout,("-debug") if ($debug);
-    push @$plugout,("-gzip_output",$gzip) if (defined $gzip);
+    push @$plugout,("-gzip_output") if ($gzip);
     push @$plugout,("-output_handle",$out) if (defined $out);
     push @$plugout,("-xslt_file",$xsltfile) if (defined $xsltfile && $xsltfile ne "");
@@ -546 +566 @@
     # gobal blocking pass may set up some metadata
     &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
-    &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    #&plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    ### section below copied from import.pl
+    if ($incremental) {
+        # equivalent to saying ($keepold && ($incremental_mode eq "all"))
+
+        &inexport::prime_doc_oid_count($exportdir);
+
+
+        # Can now work out which files were new, already existed, and have
+        # been deleted
+        
+        &inexport::new_vs_old_import_diff($export_info,$block_hash,$importdir,
+                          $exportdir,$verbosity,$incremental_mode);
+        
+        my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
+        # Filter out any in gsdl/tmp area
+        my @filtered_deleted_files = ();
+        my $gsdl_tmp_area = &util::filename_cat($ENV{'GSDLHOME'}, "tmp");
+        my $collect_tmp_area = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+        $gsdl_tmp_area = &util::filename_to_regex($gsdl_tmp_area);
+        $collect_tmp_area = &util::filename_to_regex($collect_tmp_area);
+
+
+        foreach my $df (@deleted_files) {
+        next if ($df =~ m/^$gsdl_tmp_area/);
+        next if ($df =~ m/^$collect_tmp_area/);
+        
+        push(@filtered_deleted_files,$df);
+        }
+
+        
+        @deleted_files = @filtered_deleted_files;
+
+        if (scalar(@deleted_files>0)) {
+        print STDERR "Files deleted since last import:\n  ";
+        print STDERR join("\n  ",@deleted_files), "\n";
+        }
+        
+        my @new_files = sort keys %{$block_hash->{'new_files'}};
+        if (scalar(@new_files>0)) {
+        print STDERR "New files since last import:\n  ";
+        print STDERR join("\n  ",@new_files), "\n";
+        }
+        
+        &inexport::mark_docs_for_deletion($export_info,$block_hash,\@deleted_files,
+                          $exportdir,$verbosity);
+
+        &inexport::mark_docs_for_reindex($export_info,$block_hash,
+                         $exportdir,$verbosity);
+
+        my @reindex_files = sort keys %{$block_hash->{'reindex_files'}};
+
+        if (scalar(@reindex_files>0)) {
+        print STDERR "Files to reindex since last import:\n  ";
+        print STDERR join("\n  ",@reindex_files), "\n";
+        }
+
+
+        # not sure if the following will work -- will the metadata data-structure be correctly initialized
+        # in the right order?
+#       foreach my $file (@new_files, @reindex_files) {
+#       &plugin::read ($pluginfo, $importdir, $file, $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+#       }
+
+
+        # Play it safe, and run through the entire folder, only processing new or edited files
+        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+
+    }
+    else {
+        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    }
+
+    ### end copy
     } 
     else {
@@ -553 +646 @@
         &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
     }
+
+    my @deleted_files = keys %{$manifest_lookup->{'delete'}};
+
+    &inexport::mark_docs_for_deletion($export_info,{},\@deleted_files,$exportdir);
+
     }
 
@@ -578 +676 @@
     &plugin::deinit($pluginfo, $processor);
         
+    # Store the value of OIDCount (used in doc.pm) so it can be
+    # restored correctly to this value on an incremental build
+    &inexport::store_doc_oid_count($exportdir);
+
     # write out the export information file
     #$processor->close_file_output() if $groupsize > 1;
@@ -587 +689 @@
 ##  $export_info->save_info($expinfo_doc_filename);
 #    }
-
-    my $expinfo_src_filename = &util::filename_cat ($exportdir, "archiveinf-src");   
-    &util::rename_gdbm_file($expinfo_src_filename); # ensures gdb
-    $expinfo_src_filename .= ".gdb";