Changeset 18440 for gsdl

Timestamp:

2009-02-01T14:35:22+13:00 (15 years ago)

Author:

davidb

Message:

Modifications for incremental building to support files that need to be deleted

Location:

gsdl/trunk/bin/script

Files:

• export.pl (modified) (4 diffs)
• import.pl (modified) (6 diffs)
• lucene_passes.pl (modified) (1 diff)

gsdl/trunk/bin/script/export.pl

@@ -288 +288 @@
     # other vars
     my ($configfilename, $collection, $collectcfg,
-    $export_info_filename, $export_info, 
+    $expinfo_doc_filename, $export_info, 
     $gs_mode, 
     $processor, $pluginfo);
@@ -498 +498 @@
     # the plugouts should be doing this!!
     if ($saveas eq "DSpace"){
-    $export_info_filename = &util::filename_cat ($exportdir, "contents");
+    $expinfo_doc_filename = &util::filename_cat ($exportdir, "contents");
     } elsif ($saveas =~ m/^.*METS$/ || $saveas eq "MARC" ) {
-    $export_info_filename = &util::filename_cat ($exportdir, "export.inf");
+##  $expinfo_doc_filename = &util::filename_cat ($exportdir, "export.inf");
+    my $db_ext = &util::is_little_endian() ? ".ldb" : ".bdb";
+    my $doc_db = "archiveinf-doc$db_ext";
+    $expinfo_doc_filename = &util::filename_cat ($exportdir, $doc_db);
     }
         
     $export_info = new arcinfo();
-    $export_info -> load_info ($export_info_filename);  
+    $export_info -> load_info ($expinfo_doc_filename);  
         
     my ($plugout); 
@@ -550 +553 @@
     else {
     # process any files marked for exporting
-    foreach my $file (keys %{$manifest_lookup->{'export'}}) {
+    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
         &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
     }
@@ -581 +584 @@
     #$processor->close_file_output() if $groupsize > 1;
     $processor->close_group_output() if $processor->is_group();
-    # why do we need this??
-    if ($saveas =~ m/^.*METS$/) {
-    $export_info->save_info($export_info_filename);
+
+    if (($saveas =~ m/^.*METS$/) || ($saveas eq "MARC")) {
+    # Not all export types need this (e.g. DSpace)
+    $export_info->save_info($expinfo_doc_filename);
     }
         

gsdl/trunk/bin/script/import.pl

@@ -287 +287 @@
     # other vars
     my ($configfilename, $collection, $collectcfg, 
-    $archive_info_filename, $archive_info, 
+    $arcinfo_doc_filename, $arcinfo_src_filename, $archive_info, 
     $gs_mode,
     $processor, $pluginfo);
@@ -518 +518 @@
 
     # read the archive information file
-    $archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
+##  $arcinfo_doc_filename = &util::filename_cat ($archivedir, "archives.inf");
+
+    my $db_ext = &util::is_little_endian() ? ".ldb" : ".bdb";
+    my $doc_db = "archiveinf-doc$db_ext";
+    my $src_db = "archiveinf-src$db_ext";
+    $arcinfo_doc_filename = &util::filename_cat ($archivedir, $doc_db);
+    $arcinfo_src_filename = &util::filename_cat ($archivedir, $src_db);
     
     $archive_info = new arcinfo ();
-    $archive_info->load_info ($archive_info_filename);
+    $archive_info->load_info ($arcinfo_doc_filename);
     if ($reversesort) {
     $archive_info->reverse_sort();
+    }
+
+    if ($manifest eq "") {
+    # Load in list of files in import folder from last import (if present)
+    $archive_info->load_import_filelist ($arcinfo_src_filename);
     }
 
@@ -560 +571 @@
     # gobal blocking pass may set up some metadata
     &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+    # Can now work out which files were deleted
+
+    # First convert all files to absolute form
+    # This is to support the situation where the import folder is not
+    # the default
+
+    my $prev_all_files = $archive_info->{'import_filelist'};
+    foreach my $prev_file (keys %$prev_all_files) {
+
+        if (!&util::filename_is_absolute($prev_file)) {
+        my $full_prev_file = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},$prev_file);
+        delete $prev_all_files->{$prev_file};
+        $prev_all_files->{$full_prev_file} = 1;
+        }
+    }
+
+    # Figure out which are the new files, existing files and so
+    # by implication the files from the previous import that are not
+    # there any more => mark them for deletion
+    foreach my $curr_file (keys %{$block_hash->{'all_files'}}) {
+
+        my $full_curr_file = $curr_file;
+
+        if (!&util::filename_is_absolute($curr_file)) {
+        # add in import dir to make absolute
+        $full_curr_file = &util::filename_cat($importdir,$curr_file);
+        }
+
+##      print STDERR "**** Checking $curr_file\n";
+
+        # figure of if new file or not
+        if (defined $prev_all_files->{$full_curr_file}) {
+        # had it before
+        $block_hash->{'existing_files'}->{$curr_file} = 1;
+        # Now remove it, so by end of loop only the files
+        # that need deleting are left
+
+        delete $prev_all_files->{$full_curr_file}
+        }
+        else {
+        $block_hash->{'new_files'}->{$curr_file} = 1;
+        }
+
+        delete $block_hash->{'all_files'}->{$curr_file};
+    }
+
+    print STDERR "Delete files:\n  ";
+
+    my @delete_files = keys %$prev_all_files;
+    print STDERR join("\n  ",@delete_files), "\n";
+
     &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
     }
@@ -565 +627 @@
     {
     # process any files marked for importing
-    foreach my $file (keys %{$manifest_lookup->{'import'}}) {
+    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
         &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
     }
@@ -571 +633 @@
     # record files marked for deletion in arcinfo
     foreach my $file (keys %{$manifest_lookup->{'delete'}}) {
-        # consider finding it?
+        # use 'archiveinf-src' GDBM file to look up all the OIDs
+        # this file is used in (note in most cases, it's just one OID)
+
+        # An improvement would be to have the record read
+        # into a hash array
+        # gdbmRecordToHash
+
+        my $gdbm_val 
+        = &GDBMUtil::gdbmDatabaseGet($arcinfo_src_filename,$file);
+
+        my @oids = ($gdbm_val =~ m/^<oid>(.*)$/gm);
+        foreach my $oid (@oids) {
+
+        # find out if it's an assoc file or main doc
+
+        # archiveinf-doc, lookup $oid
+        # if "doc-file" 
+        #   mark it for deletion
+
+        # else (assoc file)
+        #  mark all for re-indexing
+
+        # Now delete file 
+        }
+
         # $archive_info->add_info($OID,$doc_xml_file,"D");
     }
@@ -585 +671 @@
 
 # The following 'if' statement is in the export.pl version of the script,
-# but not (so far) the import.pl version.  Why is this?
-##    if ($saveas =~ m/^.*METS$/) {
+# The reason for the 'if' statement is now given in export.pl
+# Unclear at this point if the same should be done here
+##    if (($saveas =~ m/^.*METS$/) || ($saveas eq "MARC")) {
+    # Not all export types need this (e.g. DSpace)
+
     # should we still do this in debug mode??
 
-    $archive_info->save_info($archive_info_filename);
+    # for backwards compatability with archvies.inf file
+    if ($arcinfo_doc_filename =~ m/\.inf$/) {
+    $archive_info->save_info($arcinfo_doc_filename);
+    }
+
 ##    }
     

gsdl/trunk/bin/script/lucene_passes.pl

@@ -131 +131 @@
         $doc_xml = "";
         $output_filename = "";
+    }
+    elsif ($line =~ m/<\/Delete>\s*$/) {
+        if ($mode eq "index") {
+        $doc_xml =~ s/\n+/\n/g;
+
+        # notify lucene indexer
+        print PIPEOUT "$doc_xml";
+        }
+        $doc_xml = "";
     }
     }