Changeset 18469

Timestamp:

2009-02-06T18:19:44+13:00 (15 years ago)

Author:

davidb

Message:

Support for reindexing a document added

Location:

gsdl/trunk

Files:

• bin/script/import.pl (modified) (1 diff)
• common-src/src/gdbmedit/gdbmdel/gdbmdel.cpp (modified) (2 diffs)
• common-src/src/gdbmedit/gdbmkeys/gdbmkeys.cpp (modified) (1 diff)
• common-src/src/gdbmedit/txt2db/txt2db.cpp (modified) (3 diffs)
• perllib/GDBMUtils.pm (modified) (1 diff)
• perllib/arcinfo.pm (modified) (1 diff)
• perllib/basebuilder.pm (modified) (1 diff)
• perllib/basebuildproc.pm (modified) (7 diffs)
• perllib/classify/AZList.pm (modified) (1 diff)
• perllib/inexport.pm (modified) (8 diffs)
• perllib/plugins/ArchivesInfPlugin.pm (modified) (5 diffs)
• perllib/plugins/BasePlugin.pm (modified) (3 diffs)
• perllib/plugins/DirectoryPlugin.pm (modified) (1 diff)
• perllib/util.pm (modified) (1 diff)

gsdl/trunk/bin/script/import.pl

@@ -572 +572 @@
     # gobal blocking pass may set up some metadata
     &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
-    # Can now work out which files were new, already existed, and have
-    # been deleted
-
-    &inexport::new_vs_old_import_diff($archive_info,$block_hash,$importdir);
-    
-    my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
-    if (scalar(@deleted_files>0)) {
-        print STDERR "Delete files:\n  ";
-        print STDERR join("\n  ",@deleted_files), "\n";
-    }
-
-    my @new_files = sort keys %{$block_hash->{'new_files'}};
-    if (scalar(@new_files>0)) {
-        print STDERR "New files:\n  ";
-        print STDERR join("\n  ",@new_files), "\n";
-    }
-
-    &inexport::mark_docs_for_deletion($archive_info,\@deleted_files,$archivedir,
-        $verbosity);
+
+
+    if (!$removeold) {
+        # Can now work out which files were new, already existed, and have
+        # been deleted
+        
+        &inexport::new_vs_old_import_diff($archive_info,$block_hash,$importdir);
+        
+        my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
+        if (scalar(@deleted_files>0)) {
+        print STDERR "Files deleted since last import:\n  ";
+        print STDERR join("\n  ",@deleted_files), "\n";
+        }
+        
+        my @new_files = sort keys %{$block_hash->{'new_files'}};
+        if (scalar(@new_files>0)) {
+        print STDERR "New files since last import:\n  ";
+        print STDERR join("\n  ",@new_files), "\n";
+        }
+        
+        &inexport::mark_docs_for_deletion($archive_info,\@deleted_files,
+                          $archivedir,$verbosity);
+
+        my @existing_files = sort keys %{$block_hash->{'existing_files'}};
+
+        my @reindex_files
+        = &inexport::mark_docs_for_reindex($archive_info,\@existing_files,
+                           $archivedir,$verbosity);
+    }
 
     &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);

gsdl/trunk/common-src/src/gdbmedit/gdbmdel/gdbmdel.cpp

@@ -77 +77 @@
   // open the database
 #ifdef __WIN32__
-  dbf = gdbm_open (argv[1], block_size, GDBM_READER, 00664, NULL, 0);
+  dbf = gdbm_open (argv[1], block_size, GDBM_WRCREAT, 00664, NULL, 1);
 #else
-  dbf = gdbm_open  (argv[1], block_size, GDBM_READER, 00664, NULL);
+  dbf = gdbm_open  (argv[1], block_size, GDBM_WRCREAT, 00664, NULL);
 #endif
   if (dbf == NULL) 
     {
-      cerr << "couldn't create " << argv[1] << endl;
+      cerr << "Couldn't open " << argv[1] << endl;
       exit (-1);
     }
@@ -96 +96 @@
   //  -1 == key does not exist or there was an error
 
+  if (status < 0) {
+    cerr << "Opened database " << argv[1] << endl;
+    cerr << "  but couldn't delete: " << argv[2] << endl;
+  }
+
   gdbm_close (dbf);
 

gsdl/trunk/common-src/src/gdbmedit/gdbmkeys/gdbmkeys.cpp

@@ -82 +82 @@
 #endif
   if (dbf == NULL) {
-    cerr << "couldn't open " << argv[1] << endl;
+    cerr << "Couldn't open " << argv[1] << endl;
     exit (0);
   }

gsdl/trunk/common-src/src/gdbmedit/txt2db/txt2db.cpp

@@ -72 +72 @@
   char *dbname;
   int append = 0;
+  int delkey = 0;
+
   if (argc == 3) {
     if (strcmp (argv[1], "-append") == 0) {
@@ -105 +107 @@
     value = "";
     
-    // look for [key]\n
+    // Parse out 'key' from [key]\n
+
+    // scan for first occurrence of [
     while (!cin.eof() && c != '[') cin.get(c);
+
     if (!cin.eof()) cin.get(c); // skip [
+
+    // now look for closing ], building up 'key' as we go
     while (!cin.eof() && c != ']') {
       key.push_back ((unsigned char)c);
       cin.get(c);
     }
-    if (!cin.eof()) cin.get(c); // skip ]
+
+    if (!cin.eof()) {
+      // most likely an eol char, but if '-', then signifies record
+      // is to be deleted, not added
+      cin.get(c); 
+      if (c == '-') {   
+    delkey = 1;
+      }
+      else {
+    delkey = 0;
+      }
+    }
     while (!cin.eof() && (c == '\n' || c == '\r')) cin.get(c);
     
@@ -155 +173 @@
       }
       key_data.dsize = strlen(key_data.dptr);
+
+      if (delkey) {
+    // delete the given key
+    if (gdbm_delete(dbf, key_data) < 0) {
+      cerr << "gdbm_delete returned an error" << endl;
+    }
+      }
+      else {
+
+    // add/append
+
+    // convert value to a datum datatype
+    datum value_data;
+    value_data.dptr = value.getcstr();
+    if (value_data.dptr == NULL) {
+      cerr << "NULL value_data.dptr" << endl;
+      exit (0);
+    }
+    value_data.dsize = strlen(value_data.dptr);
       
-      // convert value to a datum datatype
-      datum value_data;
-      value_data.dptr = value.getcstr();
-      if (value_data.dptr == NULL) {
-    cerr << "NULL value_data.dptr" << endl;
-    exit (0);
-      }
-      value_data.dsize = strlen(value_data.dptr);
-      
-      // store the value
-      if (gdbm_store (dbf, key_data, value_data, GDBM_REPLACE) < 0) {
-    cerr << "gdbm_store returned an error" << endl;
-    exit (0);
-      }
-      
+    // store the value
+    if (gdbm_store (dbf, key_data, value_data, GDBM_REPLACE) < 0) {
+      cerr << "gdbm_store returned an error" << endl;
+      exit (0);
+    }
+
+    
+    free(value_data.dptr);
+      }
+
       free(key_data.dptr);
-      free(value_data.dptr);
     }
   }

gsdl/trunk/perllib/GDBMUtils.pm

@@ -93 +93 @@
 
     # Remove the document from the database
-    print STDERR "#Set document\ncmd: gdbmset$exe \"$database\" \"$oid\"\n" if $debug;
-
-    `gdbmdel$exe "$database" "$oid"`;
+
+    my $cmd = "gdbmdel$exe \"$database\" \"$oid\"";
+    print STDERR "#Delete document\ncmd: $cmd" if $debug;
+
+    `$cmd`;
+
 }
 

gsdl/trunk/perllib/arcinfo.pm

@@ -251 +251 @@
     }
 
-    $self->delete_info ($OID);
+    if (defined $self->{'info'}->{$OID}) {
+    # test to see if we are in a reindex situation
+
+    my $existing_status_info = $self->get_status_info($OID);
+
+    if ($existing_status_info eq "D") {
+        # yes, we're in a reindexing situation
+        $self->delete_info ($OID);
+
+
+        # force setting to "reindex"
+        $index_status = "R";
+
+    }
+    else {
+        # some other, possibly erroneous, situation has arisen
+        # where the document already seems to exist
+        print STDERR "Warning: $OID already exists with index status $existing_status_info\n";
+        print STDERR "         Deleting previous version\n";
+
+        $self->delete_info ($OID);
+    }
+    }
+
     $self->{'info'}->{$OID} = [$doc_file,$index_status];
     push (@{$self->{'order'}}, [$OID, $sortmeta]);
+
+
 }
 

gsdl/trunk/perllib/basebuilder.pm

@@ -424 +424 @@
 
     # Output classifier reverse lookup, used in incremental deletion
-    #&classify::print_reverse_lookup($infodb_handle);
+    ####&classify::print_reverse_lookup($infodb_handle);
 
     # output doclist

gsdl/trunk/perllib/basebuildproc.pm

@@ -343 +343 @@
 {
     my $self = shift (@_);
-    my ($field) = @_;
+    my ($field,$edit_mode) = @_;
 
     # Keep some statistics relating to metadata sets used and
@@ -355 +355 @@
     my $core_field = $2;
 
-    $self->{'doc_mdprefix_fields'}->{$prefix}->{$core_field}++;
-    $self->{'mdprefix_fields'}->{$prefix}->{$core_field}++;
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+        $self->{'doc_mdprefix_fields'}->{$prefix}->{$core_field}++;
+        $self->{'mdprefix_fields'}->{$prefix}->{$core_field}++;
+    }
+    else {
+        # delete
+        $self->{'doc_mdprefix_fields'}->{$prefix}->{$core_field}--;
+        $self->{'mdprefix_fields'}->{$prefix}->{$core_field}--;
+    }
+
     }
     elsif ($field =~ m/^[[:upper:]]/) {
     # implicit 'ex' metadata set
 
-    $self->{'doc_mdprefix_fields'}->{'ex'}->{$field}++;
-    $self->{'mdprefix_fields'}->{'ex'}->{$field}++;
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+
+        $self->{'doc_mdprefix_fields'}->{'ex'}->{$field}++;
+        $self->{'mdprefix_fields'}->{'ex'}->{$field}++;
+    }
+    else {
+        # delete
+        $self->{'doc_mdprefix_fields'}->{'ex'}->{$field}--;
+        $self->{'mdprefix_fields'}->{'ex'}->{$field}--;
+    }
     }
 
@@ -425 +441 @@
     # delete
     $self->{'num_docs'} -= 1 unless ($doctype eq "classification");
-    return;
     }
 
@@ -448 +463 @@
 
     # update a few statistics
-    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
-    $self->{'num_sections'} += 1 unless ($doctype eq "classification");
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+
+        $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+        $self->{'num_sections'} += 1 unless ($doctype eq "classification");
+    }
+    else {
+        # delete
+        $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
+        $self->{'num_sections'} -= 1 unless ($doctype eq "classification");
+    }
 
     # output the fact that this document is a document (unless doctype
@@ -488 +511 @@
         # special case for URL metadata
         if ($field =~ /^URL$/i) {
-            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value, { 'section' => [ $section_OID ] });
+            if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+
+            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value, { 'section' => [ $section_OID ] });
+            }
+            else {
+            # delete
+            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value);
+            }
+
+            
         }
 
@@ -496 +528 @@
             if ($section eq "" && $self->{'store_metadata_coverage'} =~ /^true$/i)
             {
-            $self->infodb_metadata_stats($field);
+            $self->infodb_metadata_stats($field,$edit_mode);
             }
         }
@@ -565 +597 @@
     } 
     
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID, \%section_infodb);
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+
+        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID, \%section_infodb);
+    }
+    else {
+        # delete
+        &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID);
+    }
+
     
     # output a database entry for the document number, except for Lucene (which no longer needs this information)
     unless (ref($self) eq "lucenebuildproc")
     {
-        if ($self->{'db_level'} eq "document") {
-        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'}, { 'section' => [ $doc_OID ] });
+        if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+        
+        if ($self->{'db_level'} eq "document") {
+            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'}, { 'section' => [ $doc_OID ] });
+        }
+        else {
+            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'}, { 'section' => [ $section_OID ] });
+        }
         }
         else {
-        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'}, { 'section' => [ $section_OID ] });
+
+        if ($self->{'db_level'} eq "document") {
+            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'});
+        }
+        else {
+            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'});
+        }
+
         }
     }

gsdl/trunk/perllib/classify/AZList.pm

@@ -124 +124 @@
     my $outhandle = $self->{'outhandle'};
 
-    if ($edit_mode eq "delete") {
+    if (($edit_mode eq "delete") || ($edit_mode eq "reindex")) {
     $self->oid_hash_delete($doc_OID,'list');
-    return;
+    return if ($edit_mode eq "delete");     
     }
 

gsdl/trunk/perllib/inexport.pm

@@ -27 +27 @@
 
 use strict;
+
+use File::Basename;
 
 use util;
@@ -53 +55 @@
     }
 
+
     # Figure out which are the new files, existing files and so
     # by implication the files from the previous import that are not
@@ -68 +71 @@
     }
 
+    if (defined $block_hash->{'file_blocks'}->{$full_curr_file}) {
+        # If in block list, we want to ignore it
+        delete $block_hash->{'all_files'}->{$curr_file};
+
+        if (defined $full_prev_all_files->{$full_curr_file}) {
+        # also make sure it is gone from 'previous' list so
+        # not mistaken for a file that needs to be deleted
+        delete $full_prev_all_files->{$full_curr_file};
+        }
+        next;
+    }
+
     # figure of if new file or not
     if (defined $full_prev_all_files->{$full_curr_file}) {
+        
         # had it before
-        $block_hash->{'existing_files'}->{$curr_file} = 1;
+        $block_hash->{'existing_files'}->{$full_curr_file} = 1;
+
         # Now remove it, so by end of loop only the files
         # that need deleting are left
@@ -78 +95 @@
         }
     else {
-        $block_hash->{'new_files'}->{$curr_file} = 1;
+        $block_hash->{'new_files'}->{$full_curr_file} = 1;
     }
     
@@ -84 +101 @@
     }
 
-    # By this point full_prev_all_files contains only the files
-    # that are not in the current import folder => i.e. files
-    # to be deleted
+    # By this point full_prev_all_files contains the files
+    # mentioned in archiveinf-src.db but are not in the 'import'
+    # folder (or whatever was specified through -importdir ...)
+
+    # This list can contain files that were created in the 'tmp' or
+    # 'cache' areas (such as screen-size and thumbnail images).
     #
-    # The value in each key is its "local" import file name, which is what 
-    # we want to use
+    # In building the final list of files to delete, we test to see if
+    # it exists on the filesystem and if it does (unusual for a file
+    # that's allegedly deleted!) , supress it from going into the final
+    # list
+
+    my $collectdir = $ENV{'GSDLCOLLECTDIR'};
+
     my @deleted_files = values %$full_prev_all_files;
-    map { $block_hash->{'deleted_files'}->{$_} = 1 } @deleted_files;
+    map { my $curr_file = $_;
+      my $full_curr_file = $curr_file;
+
+      if (!&util::filename_is_absolute($curr_file)) {
+          # add in import dir to make absolute
+
+          $full_curr_file = &util::filename_cat($collectdir,$curr_file);
+      }
+
+
+      if (!-e $full_curr_file) {
+          $block_hash->{'deleted_files'}->{$curr_file} = 1;
+      }
+      } @deleted_files;
 }
 
@@ -110 +148 @@
     # this file is used in (note in most cases, it's just one OID)
     
-    # An improvement would be to have the record read
-    # into a hash array
     my $src_rec = GDBMUtils::gdbmRecordToHash($arcinfo_src_filename,$file);
     my $oids = $src_rec->{'oid'};
     foreach my $oid (@$oids) {
 
-        # find out if it's an assoc file or main doc
+        # Find out if it's an assoc file or main doc
 
         my $doc_rec = GDBMUtils::gdbmRecordToHash($arcinfo_doc_filename,$oid);
-##      print STDERR "file = $file\n";
-
         if ($doc_rec->{'src-file'}->[0] eq $file) {
-        # mark it for deletion
+        # It's the main doc
+        # => mark it for deletion
+    
         if ($verbosity>1) {
-            print STDERR "$oid marked to be deleted\n";
+            print STDERR "$oid marked to be deleted from index on next buildcol.pl\n";
         }
         $archive_info->set_status_info($oid,"D");
 
         my $val = &GDBMUtils::gdbmDatabaseGet($arcinfo_doc_filename,$oid);
-        $val =~ s/^<index-status>(.*)$/<index-status>D/m;
-        &GDBMUtils::gdbmDatabaseSet($arcinfo_doc_filename,$oid,$val);
+        my ($index_status) = ($val =~ m/^<index-status>(.*)$/m);
+        if ($index_status ne "D") {
+            $val =~ s/^<index-status>(.*)$/<index-status>D/m;
+            &GDBMUtils::gdbmDatabaseSet($arcinfo_doc_filename,$oid,$val);
+            my $doc_file = $doc_rec->{'doc-file'}->[0];
+
+            my $doc_filename = &util::filename_cat($archivedir,$doc_file);
+
+
+            my ($doc_tailname, $doc_dirname, $suffix) 
+            = File::Basename::fileparse($doc_filename, "\\.[^\\.]+\$");
+
+            print STDERR "Removing $doc_dirname\n" if ($verbosity>2);
+
+            &util::rm_r($doc_dirname);
+            
+        }
         }
         else {
@@ -149 +200 @@
         }
         }
+
+        GDBMUtils::gdbmDatabaseRemove($arcinfo_src_filename,$file);
     }
     }
@@ -155 +208 @@
 
 
+sub mark_docs_for_reindex
+{
+    my ($archive_info,$existing_files_ref,$archivedir,$verbosity) = @_;
+
+    # Reindexing is accomplished by deleting the previously indexed
+    # version of the document, and then allowing the new version to
+    # be indexed (as would a new document be indexed). 
+    # 
+    # The first step (marking for deletion) is implemented by this routine.
+    # 
+    # By default in Greenstone a new version of an index will hash to
+    # a new unique OID, and the above strategy of reindex=delete+add
+    # works fine.  A special case arises when a persistent OID is 
+    # allocated to a document (for instance through a metadata field),
+    # and the second step to reindexing (see XXXX) detects this and
+    # deals with it appropriately.
+
+    my $db_ext = &util::is_little_endian() ? ".ldb" : ".bdb";
+    my $doc_db = "archiveinf-doc$db_ext";
+    my $arcinfo_doc_filename = &util::filename_cat ($archivedir, $doc_db);
+
+
+    my $archiveinf_timestamp = -M $arcinfo_doc_filename;
+
+    my $reindex_files_ref = [];
+
+    foreach my $existing_filename (@$existing_files_ref) {
+    
+    if (-M $existing_filename < $archiveinf_timestamp) {
+        # file is newer than last build
+        
+        my $existing_file = $existing_filename;
+        my $collectdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'});
+
+        $existing_file =~ s/^$collectdir(\\|\/)?//;
+        
+        print STDERR "**** Deleting existing file: $existing_file\n";
+
+        push(@$reindex_files_ref,$existing_file);
+    }
+
+    }
+    
+    mark_docs_for_deletion($archive_info,$reindex_files_ref,$archivedir,$verbosity);
+
+    return @$reindex_files_ref;
+}
+
+
+
 1;

gsdl/trunk/perllib/plugins/ArchivesInfPlugin.pm

@@ -90 +90 @@
 
     if (defined $archive_info) {
-    print STDERR "********* have parsed and processed an archive info file\n";
-
     my $archive_info_filename = $self->{'archive_info_filename'};
 
@@ -100 +98 @@
 
         my $index_status = $archive_info->get_status_info($doc_oid);
-        print STDERR "*** Updating $doc_oid $index_status\n";
+####        print STDERR "*** Away to Update $doc_oid $index_status\n";
 
         if ($index_status eq "D") {
         # delete
         $archive_info->delete_info($doc_oid);
+        &GDBMUtils::gdbmDatabaseRemove($archive_info_filename,$doc_oid);
         }
         elsif ($index_status =~ m/^(I|R)$/) {
@@ -215 +214 @@
         my $process_file = 1;
 
-        # ...unless the build processor is incremental capable and -incremental was specified
+        # ...unless the build processor is incremental capable and -incremental was specified, in which case we need to check its index_status flag
         if ($processor->is_incremental_capable() && $self->{'incremental'})
         {
@@ -230 +229 @@
         }
         elsif ($index_status eq "R") {
-            # Need to be delete it from the index.
+            # Need to be reindexed/replaced
             $new_mode = $curr_mode."reindex";
+
             $process_file = 1;
         }
@@ -237 +237 @@
         # ... or we're being asked to delete it (in which case skip it)
         elsif ($index_status eq "D") {
-        # Delete it somehow from archives dir!!
-        # => get short name, lop off filename, concat archivedir
-        # move to recyle bin
+        # Non-incremental Delete
+        # It's already been deleted from the archives directory
+        # (done during import.pl)
+        # => All we need to do here is not process it
 
         $process_file = 0;
         }
+
+        if (!$processor->is_incremental_capable() && $self->{'incremental'}) {
+        # Nag feature
+        if (!defined $self->{'incremental-warning'}) {
+            print $outhandle "\n";
+            print $outhandle "Warning: command-line option '-incremental' used with *non-incremental*\n";
+            print $outhandle "         processor '", ref $processor, "'. Some conflicts may arise.\n";
+            print $outhandle "\n";
+            sleep 10;
+            $self->{'incremental-warning'} = 1;
+        }
+        }
+
+
 
         if ($process_file) {

gsdl/trunk/perllib/plugins/BasePlugin.pm

@@ -833 +833 @@
 
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
     # create a new document
     my $doc_obj = new doc ($filename_full_path, "indexed_doc", $self->{'file_rename_method'});
@@ -840 +841 @@
     $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
  
+
     # sets the UTF8 filename (Source) for display and sets the url ref to URL encoded version
     # of the UTF8 filename (SourceFile) for generated files
     $self->set_Source_metadata($doc_obj, $filename_no_path);
+
 
     # plugin specific stuff - what args do we need here??
@@ -946 +949 @@
     # delete any temp files that we may have created
     $self->clean_up_after_doc_obj_processing();
+
 
     # if process_status == 1, then the file has been processed.

gsdl/trunk/perllib/plugins/DirectoryPlugin.pm

@@ -575 +575 @@
 
     if (defined $self->{'inf_timestamp'}) {
-        my $inf_timestamp = $self->{'inf_timestamp'};
-
-        if (! -d $full_filename) {
-        my $filename_timestamp = -M $full_filename;
-        if ($filename_timestamp > $inf_timestamp) {
-            # filename has been around for longer than inf
-            print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
-            next;
+        # Look to see if it's a completely new file
+
+        if (!$block_hash->{'new_files'}->{$full_filename}) {
+        # Not a new file, must be an existing file
+        # Let' see if it's newer than the last import.pl
+
+
+        my $inf_timestamp = $self->{'inf_timestamp'};
+
+        if (! -d $full_filename) {
+            my $filename_timestamp = -M $full_filename;
+            if ($filename_timestamp > $inf_timestamp) {
+            # filename has been around for longer than inf
+            print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
+            next;
+            }
+            else {
+            # Remove old folder in archives (might hash to something different)
+            # *** should be doing this on a Del one as well
+            # but leave folder name?? and ensure hashs to
+            # same again??
+
+            # Then let through as new doc??
+
+            # mark to doc-oids that rely on it for re-indexing
+            ## &GDBMUtils::gdbmDatabase();
+
+            }
         }
         }

gsdl/trunk/perllib/util.pm

@@ -35 +35 @@
 sub rm {
     my (@files) = @_;
+
     my @filefiles = ();