Changeset 18456 for gsdl/trunk/bin/script/import.pl

Timestamp:

2009-02-03T09:48:19+13:00 (15 years ago)

Author:

davidb

Message:

Additions to support the deleting of documents from the index. Only works for indexers that support incremental building, e.g. lucene

File:

• gsdl/trunk/bin/script/import.pl (modified) (4 diffs)

gsdl/trunk/bin/script/import.pl

@@ -63 +63 @@
 use plugout;
 use manifest;
+use inexport;
 use util;
 use scriptutil;
@@ -534 +535 @@
     if ($manifest eq "") {
     # Load in list of files in import folder from last import (if present)
-    $archive_info->load_import_filelist ($arcinfo_src_filename);
+    $archive_info->load_prev_import_filelist ($arcinfo_src_filename);
     }
 
@@ -571 +572 @@
     # gobal blocking pass may set up some metadata
     &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
-    # Can now work out which files were deleted
-
-    # First convert all files to absolute form
-    # This is to support the situation where the import folder is not
-    # the default
-
-    my $prev_all_files = $archive_info->{'import_filelist'};
-    foreach my $prev_file (keys %$prev_all_files) {
-
-        if (!&util::filename_is_absolute($prev_file)) {
-        my $full_prev_file = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},$prev_file);
-        delete $prev_all_files->{$prev_file};
-        $prev_all_files->{$full_prev_file} = 1;
-        }
-    }
-
-    # Figure out which are the new files, existing files and so
-    # by implication the files from the previous import that are not
-    # there any more => mark them for deletion
-    foreach my $curr_file (keys %{$block_hash->{'all_files'}}) {
-
-        my $full_curr_file = $curr_file;
-
-        if (!&util::filename_is_absolute($curr_file)) {
-        # add in import dir to make absolute
-        $full_curr_file = &util::filename_cat($importdir,$curr_file);
-        }
-
-##      print STDERR "**** Checking $curr_file\n";
-
-        # figure of if new file or not
-        if (defined $prev_all_files->{$full_curr_file}) {
-        # had it before
-        $block_hash->{'existing_files'}->{$curr_file} = 1;
-        # Now remove it, so by end of loop only the files
-        # that need deleting are left
-
-        delete $prev_all_files->{$full_curr_file}
-        }
-        else {
-        $block_hash->{'new_files'}->{$curr_file} = 1;
-        }
-
-        delete $block_hash->{'all_files'}->{$curr_file};
-    }
-
-    print STDERR "Delete files:\n  ";
-
-    my @delete_files = keys %$prev_all_files;
-    print STDERR join("\n  ",@delete_files), "\n";
+    # Can now work out which files were new, already existed, and have
+    # been deleted
+
+    &inexport::new_vs_old_import_diff($archive_info,$block_hash,$importdir);
+    
+    my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
+    if (scalar(@deleted_files>0)) {
+        print STDERR "Delete files:\n  ";
+        print STDERR join("\n  ",@deleted_files), "\n";
+    }
+
+    my @new_files = sort keys %{$block_hash->{'new_files'}};
+    if (scalar(@new_files>0)) {
+        print STDERR "New files:\n  ";
+        print STDERR join("\n  ",@new_files), "\n";
+    }
+
+    &inexport::mark_docs_for_deletion($archive_info,\@deleted_files,$archivedir,
+        $verbosity);
 
     &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
     }
-    else
+                   else
     {
     # process any files marked for importing
@@ -631 +601 @@
     }
 
-    # record files marked for deletion in arcinfo
-    foreach my $file (keys %{$manifest_lookup->{'delete'}}) {
-        # use 'archiveinf-src' GDBM file to look up all the OIDs
-        # this file is used in (note in most cases, it's just one OID)
-
-        # An improvement would be to have the record read
-        # into a hash array
-        # gdbmRecordToHash
-
-        my $gdbm_val 
-        = &GDBMUtil::gdbmDatabaseGet($arcinfo_src_filename,$file);
-
-        my @oids = ($gdbm_val =~ m/^<oid>(.*)$/gm);
-        foreach my $oid (@oids) {
-
-        # find out if it's an assoc file or main doc
-
-        # archiveinf-doc, lookup $oid
-        # if "doc-file" 
-        #   mark it for deletion
-
-        # else (assoc file)
-        #  mark all for re-indexing
-
-        # Now delete file 
-        }
-
-        # $archive_info->add_info($OID,$doc_xml_file,"D");
-    }
+    my @deleted_files = keys %{$manifest_lookup->{'delete'}};
+
+    &inexport::mark_docs_for_deletion($archive_info,\@deleted_files,$archivedir);
     }