Index: /main/trunk/greenstone2/perllib/inexport.pm
===================================================================
--- /main/trunk/greenstone2/perllib/inexport.pm	(revision 37151)
+++ /main/trunk/greenstone2/perllib/inexport.pm	(revision 37152)
@@ -47,4 +47,7 @@
 use parse2;
 
+use DocHistoryFileUtils;
+use FileUtils;
+
 use File::Basename;
 
@@ -121,4 +124,9 @@
        'reqd' => "no",
        'hiddengli' => "yes" },
+     { 'name' => "replaceold",
+       'desc' => "{import.replaceold}",
+       'type' => "flag",
+       'reqd' => "no",
+       'hiddengli' => "yes" },
      { 'name' => "removeold",
        'desc' => "{import.removeold}",
@@ -428,6 +436,9 @@
 	$archivedir = &FileUtils::sanitizePath($archivedir);
     }
+
+    my $archivedir_keepold = "${archivedir}_keepold"; # used when file-level document-version history is in play
     $self->{'archivedir'} = $archivedir;
-
+    $self->{'archivedir_keepold'} = $archivedir_keepold; 
+    
     if (defined $self->{'default_verbosity'}) {
 	if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
@@ -478,11 +489,12 @@
     my $checkdir = ($inexport_mode eq "import") ? "archives" : "export";
 
-    my ($removeold, $keepold, $incremental, $incremental_mode) 
-	= &scriptutil::check_removeold_and_keepold($self->{'removeold'}, $self->{'keepold'}, 
-						   $self->{'incremental'}, $checkdir, 
-						   $collectcfg);
+    my ($removeold, $keepold, $replaceold, $incremental, $incremental_mode) 
+	= &scriptutil::check_removeold_keepold_replaceold($self->{'removeold'}, $self->{'keepold'}, $self->{'replaceold'},
+							  $self->{'incremental'}, $checkdir, 
+							  $collectcfg);
 
     $self->{'removeold'}        = $removeold;
     $self->{'keepold'}          = $keepold;
+    $self->{'replaceold'}       = $replaceold;
     $self->{'incremental'}      = $incremental;
     $self->{'incremental_mode'} = $incremental_mode;
@@ -507,5 +519,8 @@
 
     my $importdir   = $self->{'importdir'};
-    my $archivedir = $self->{'archivedir'} || $self->{'exportdir'};
+    my $archivedir  = $self->{'archivedir'} || $self->{'exportdir'};
+    # 'archivedir' is a tad abused, and is sometimes set to the 'exportdir' value,
+    # meaining 'archivedir_keepold' is actually the export dir name with '_keepold' appended
+    my $archivedir_keepold  = $self->{'archivedir_keepold'}; 
 
     my $incremental = $self->{'incremental'};
@@ -515,4 +530,5 @@
 
     my $removeold   = $self->{'removeold'};
+    my $replaceold  = $self->{'replaceold'};
     my $keepold     = $self->{'keepold'};
 
@@ -551,5 +567,5 @@
     }
 
-    my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
+    my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir); 
     if ($self->{'manifest'} ne "") {
 	my $manifest_filename = $self->{'manifest'};
@@ -590,8 +606,39 @@
     }
 
-    # remove the old contents of the archives directory (and tmp
-    # directory) if needed
-
-    if ($removeold) {
+    # Whether -removeold, -keepold or -replaceold there should never be an existing archivedir_keepold
+    # => Taken to be a sign of a previous import/export that has gone wrong
+    # => Print out error message and stop!
+    
+    if (&FileUtils::directoryExists($archivedir_keepold)) {
+	my $rkr_old_minus_option = undef; # rkr = remove, keep, replace (whichever one is being used)
+	if ($removeold) {
+	    $rkr_old_minus_option = "-removeold";
+	}
+	elsif ($keepold) {
+	    $rkr_old_minus_option = "-keepold";
+	}
+	elsif ($replaceold) {
+	    $rkr_old_minus_option = "-replaceold";
+	}
+	
+	&gsprintf(STDERR, "\n");
+	&gsprintf(STDERR, "Detected existing directory:\n\n");
+	&gsprintf(STDERR, "    $archivedir_keepold\n\n");
+	&gsprintf(STDERR, "Stopping $inexport_mode.\n\n");
+	
+	&gsprintf(STDERR, "**** When building with $rkr_old_minus_option, there cannot be a pre-existing 'archives_keepold' directory\n");
+	&gsprintf(STDERR, "****\n");
+	&gsprintf(STDERR, "**** Review your collection directory folder, and determine whether to:\n");
+	&gsprintf(STDERR, "****   (a) move your 'archives_keepold' back to being 'archives'; or\n");
+	&gsprintf(STDERR, "****   (b) remove your 'archives_keepold'\n");
+	&gsprintf(STDERR, "**** before running your $inexport_mode command again\n\n");
+	
+	exit 1; # c errno for 'operation not permitted'
+    }
+
+
+    # remove the old contents of the archives directory (and tmp directory) if needed
+
+    if ($removeold) {	
 	if (&FileUtils::directoryExists($archivedir)) {
 	    &gsprintf($out, "{import.removing_archives}\n");
@@ -605,4 +652,29 @@
 	    &FileUtils::removeFilesRecursive($tmpdir);
 	}
+    }
+    else {	
+	# If not $removeold, then must be $keepold or $replaceold
+	# => for either case want to "hard-link"/copy 'archives' to 'archives_keepold'
+
+	# Want to be super careful about doing this, so as not to accidentally
+	# wipe out any previous file-level document-version history
+
+	# If got to here, then there is no pre-existing $archivedir_keepold
+	# => Hard-link copy the contents of 'archives' to 'archives_keepold'
+	# => Stop if there is any issue with creating the hard-link copy
+	
+	if (!&FileUtils::hardlinkFilesRefRecursive([$archivedir],$archivedir_keepold, { 'strict' => 1 } )) {
+	    
+	    &gsprintf(STDERR, "\nError message: $!\n\n");
+	    
+	    &gsprintf(STDERR, "**** Failed to make a hard-link copy of:\n");
+	    &gsprintf(STDERR, "****     $archivedir\n");
+	    &gsprintf(STDERR, "**** to:\n");
+	    &gsprintf(STDERR, "****     $archivedir_keepold\n");
+	    &gsprintf(STDERR, "****\n");
+	    &gsprintf(STDERR, "**** Unable to proceed with file-level document-version history $inexport_mode => Stopping\n");
+
+	    exit $!;
+	}	
     }
 
@@ -627,8 +699,10 @@
     my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir, $perform_firsttime_init);
 
+
     my $archive_info = new arcinfo ($collectcfg->{'infodbtype'});
-    $archive_info->load_info ($arcinfo_doc_filename);
-    # load in rev info so we don't overwrite existing info when we do incremental import
-    # from here on, make all changes to this object, then write out the file at the end.
+    $archive_info->load_info($arcinfo_doc_filename);
+    # Load in reverse-lookup info (used to determine the docs that a file in import are used in),
+    #   so we don't overwrite existing info when we do incremental import
+    # From here on, make all changes to this object, then write out the file at the end.
     $archive_info->load_rev_info($arcinfo_src_filename);
 
@@ -710,6 +784,6 @@
     $block_hash->{'new_files'} = {};
     $block_hash->{'reindex_files'} = {};
-    # all of these are set somewhere else, so it's more readable to define them
-    # here [jmt12]
+
+    # all of these are set somewhere else, so it's more readable to define them here [jmt12]
     $block_hash->{'all_files'} = {};
     $block_hash->{'deleted_files'} = {};
@@ -829,4 +903,5 @@
 
 	my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir);
+	
 	# need to check this file exists before trying to read it - in the past
 	# it wasn't possible to have a manifest unless keepold was also set so
@@ -902,5 +977,5 @@
 	    # Can now work out which files were new, already existed, and have
 	    # been deleted
-	    
+
 	    new_vs_old_import_diff($archive_info,$block_hash,$importdir,
 				   $archivedir,$verbosity,$incremental_mode);
@@ -1007,5 +1082,5 @@
     if ($self->{'OIDtype'} eq 'incremental')
     {
-	store_doc_oid_count($archivedir);
+	store_doc_oid_count($archivedir); 
     }
 
@@ -1014,9 +1089,13 @@
     $processor->end();
     
-    #    if ($inexport_mode eq "import") {
     if ($self->{'generate_auxiliary_files'}) {
+	    
 	# write out the archive information file
 	# for backwards compatability with archvies.inf file
 	if ($arcinfo_doc_filename =~ m/(contents)|(\.inf)$/) {
+	    # In the days of this being a text file, this all we had to do
+	    # Note, if still using this form of archive-inf, then neither
+	    # incremental building nor files-level document-version history
+	    # is suported
 	    $archive_info->save_info($arcinfo_doc_filename);
 	}
@@ -1025,4 +1104,17 @@
 	}
     }
+
+
+    #
+    # Now deal with any file-level document-version history (fldv-history)
+    #
+
+    if ($keepold || $removeold) {
+
+	&DocHistoryFileUtils::archivedir_keepold_to_archivedir($collectcfg, $keepold, $replaceold, $incremental_mode, $archive_info, $archivedir,$archivedir_keepold);
+	
+    }
+    
+    
     return $pluginfo;
 }
@@ -1050,4 +1142,12 @@
     elsif ($manifest eq '' || $self->{'manifest_version'} == 1)
     {
+	#print STDERR "**** perform_process_files(): importdir=$importdir\n";
+	#print STDERR "**** block_hash:\n  ", join("\n  ", keys %{$block_hash}), "\n\n";
+	#print STDERR "**** block_hash->all_files:\n  ", join("\n  ", keys %{$block_hash->{'all_files'}}), "\n\n";
+	#print STDERR "**** block_hash->reindex_files:\n  ", join("\n  ", keys %{$block_hash->{'reindex_files'}}), "\n\n";
+
+	#print STDERR "**** block_hash->existing_files:\n  ", join("\n  ", keys %{$block_hash->{'existing_files'}}), "\n\n";
+	#print STDERR "**** block_hash->file_blocks:\n  ", join("\n  ", keys %{$block_hash->{'file_blocks'}}), "\n\n";
+
 	&plugin::read ($pluginfo, $importdir, '', $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
     }
@@ -1197,4 +1297,5 @@
     
     my $prev_all_files = $archive_info->{'prev_import_filelist'};
+    
     my $full_prev_all_files = {};
 
@@ -1228,4 +1329,6 @@
 	}
 
+	###print STDERR "*** new vs old: look to see if full_curr_file=$full_curr_file in full_prev_all_files hashmap\n";
+	
 	# figure out if new file or not
 	if (defined $full_prev_all_files->{$full_curr_file}) {
@@ -1335,5 +1438,5 @@
 	    #$existing_file =~ s/^$collectdir_resafe(\\|\/)?//;
 	    
-	    print STDERR "**** Reindexing existing file: $existing_file\n";
+	    # print STDERR "**** Reindexing existing file: $existing_file\n";
 
 	    push(@$reindex_files,$existing_file);
Index: /main/trunk/greenstone2/perllib/scriptutil.pm
===================================================================
--- /main/trunk/greenstone2/perllib/scriptutil.pm	(revision 37151)
+++ /main/trunk/greenstone2/perllib/scriptutil.pm	(revision 37152)
@@ -32,11 +32,16 @@
 use gsprintf 'gsprintf';
 
-# returns $removeold, $keepold
+# returns ($removeold, $keepold, $incremental, $incremental_mode)
 sub check_removeold_and_keepold {
 
     my ($removeold, $keepold, $incremental, $dir, $collectcfg) = @_;    
 
-    if (($keepold && $removeold) || ($incremental && $removeold) ) {
-	gsprintf(STDERR, "{scripts.both_old_options}\n", $dir);
+    if ($keepold && $removeold) {
+	gsprintf(STDERR, "{scripts.only_one_old_option}\n");
+	exit(2)	
+    } 
+
+    if ($incremental && $removeold) {
+	gsprintf(STDERR, "{scripts.inc_remove_conflict}\n", $dir);
 	sleep(3); #just in case
 	return (1,0,0,"none"); 
@@ -44,4 +49,5 @@
     } 
 
+    
     # Incremental mode may be set to "none", "onlyadd" or "all"
     # depending on status of -keepold and -incremental flags
@@ -80,3 +86,96 @@
 }
 
+
+# returns ($removeold, $keepold, $replaceold, $incremental, $incremental_mode)
+sub check_removeold_keepold_replaceold {
+
+    my ($removeold, $keepold, $replaceold, $incremental, $dir, $collectcfg) = @_;    
+
+    my $old_count = 0;
+    $old_count++ if $removeold;
+    $old_count++ if $keepold;
+    $old_count++ if $replaceold;
+
+    if ($old_count>1) {
+	gsprintf(STDERR, "{scripts.only_one_old_option}\n");
+	exit(2);
+    } 
+
+    if (($incremental && $removeold) ) {
+	gsprintf(STDERR, "{scripts.inc_remove_conflict}\n", $dir);
+	sleep(5); #just in case
+	return (1,0,0,0,"none"); 	
+    } 
+
+    # Determine what the internal 'incremental_mode' is:
+    # => May be set to "none", "onlyadd" or "all"
+    # Based on status of (-keepold|-removeold) and -incremental flags
+    #
+    # With the introduction of file-level document-version (fldv) history, the chosen name 'onlyadd'
+    # for when '-keepold' is on is a bit misleading. However, it does still get us "over the line"
+    # in terms of how it functionally operates.  In the case where pre-existing content is
+    # still in the 'import' folder, then when everything in 'archives_keepold' gets copied
+    # back, any pre-existing documents from import (which will have resulted in a doc folder
+    # in 'archives') will trigger a file-level document-version history folder inside it.  For any
+    # content that was new in 'import', it won't have a pre-existing folder inside 'archives'
+    # and so will appear as a new folder with *no* file-level document-version history folder
+    # inside it (effectively why the keepold incremental mode was originally called 'onlyadd').
+
+    my $incremental_mode = "none";
+    if ($incremental) {
+	$incremental_mode = "all";
+    } elsif ($keepold || $replaceold) {
+	$incremental_mode = "onlyadd";
+    }
+
+    if (!$keepold && !$removeold && !$replaceold && !$incremental && defined $collectcfg) {
+	# we only look at config file options if we don't have these on the command line
+	if (defined $collectcfg->{'removeold'} && $collectcfg->{'removeold'} =~ /^true$/i ) {
+	    $removeold = 1;
+	} elsif (defined $collectcfg->{'keepold'} && $collectcfg->{'keepold'} =~ /^true$/i) {
+	    $keepold = 1;
+	    $incremental_mode = "onlyadd";
+	} elsif (defined $collectcfg->{'replaceold'} && $collectcfg->{'replaceold'} =~ /^true$/i) {
+	    $replaceold = 1;
+	    $incremental_mode = "onlyadd";
+	} elsif (defined $collectcfg->{'incremental'} && $collectcfg->{'incremental'} =~ /^true$/i) {
+	    $incremental = 1;
+	    $incremental_mode = "all";
+	}
+
+	# Go through the same checks as before
+	my $cfg_old_count = 0;
+	$cfg_old_count++ if $removeold;
+	$cfg_old_count++ if $keepold;
+	$cfg_old_count++ if $replaceold;
+
+	if ($cfg_old_count>1) {
+	    gsprintf(STDERR, "{scripts.only_one_old_option}\n");
+	    exit(2);
+	}
+
+	if (($incremental && $removeold) ) {
+	    gsprintf(STDERR, "{scripts.inc_remove_conflict}\n", $dir);
+	    sleep(5); #just in case
+	    return (1,0,0,0,"none"); 	
+	} 
+    }
+
+    # default to -removeold if nothing specified
+    if (!$keepold && !$removeold && !$replaceold && !$incremental) {
+	gsprintf(STDERR, "{scripts.no_old_options} \n", $dir);
+	sleep(5); #just in case
+	return (1,0,0,0,"none");
+    }
+    
+    # incremental implies keepold
+    if ($incremental) {
+	$keepold = 1;
+    }
+    return ($removeold, $keepold, $replaceold, $incremental, $incremental_mode);
+
+}
+
+
+
 1;
Index: /main/trunk/greenstone2/perllib/strings.properties
===================================================================
--- /main/trunk/greenstone2/perllib/strings.properties	(revision 37151)
+++ /main/trunk/greenstone2/perllib/strings.properties	(revision 37152)
@@ -62,4 +62,8 @@
 
 scripts.both_old_options:WARNING: -removeold was specified with -keepold or -incremental, defaulting to -removeold. Current contents of %s directory will be deleted.
+
+scripts.inc_remove_conflict:WARNING: -incremental and -removeold were specified. Defaulting to -removeold. Current contents of %s directory will be deleted.
+
+scripts.only_one_old_option:Error: conflicting 'old' options: can only specify one of -removeold, -keepold, -replaceold. Exiting.
 
 scripts.no_old_options:WARNING: None of -removeold, -keepold or -incremental were specified, defaulting to -removeold. Current contents of %s directory will be deleted.
