Index: /main/trunk/greenstone2/perllib/DocHistoryFileUtils.pm
===================================================================
--- /main/trunk/greenstone2/perllib/DocHistoryFileUtils.pm	(revision 37149)
+++ /main/trunk/greenstone2/perllib/DocHistoryFileUtils.pm	(revision 37149)
@@ -0,0 +1,331 @@
+###########################################################################
+#
+# DocHistoryFileUtils.pm
+#  -- supporting routines for file-level document-version history (fldv-history)
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2023 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package DocHistoryFileUtils;
+
+# Pragma
+use strict;
+use warnings;
+
+# use FileHandle;
+
+# Greenstone modules
+use dbutil;
+use FileUtils;
+
+my $fldv_history_dir = "_fldv_history";
+
+sub prepend_document_version
+{
+    my ($keepold_doc_dirname,$doc_dirname) = @_;
+
+    my $status_ok = 1;
+    
+    my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$fldv_history_dir);
+
+    if (&FileUtils::directoryExists($doc_fldv_history_dirname)) {
+	# need to shuffle nminus-1, nminus-2 down by one
+	my $matching_dirs = &FileUtils::readDirectoryFiltered($doc_fldv_history_dirname,undef,"^nminus-\\d+\$");
+	
+	my @sorted_matching_dirs = sort {
+	    my ($a_num) = ($a =~ m/(\d+)$/);
+	    my ($b_num) = ($b =~ m/(\d+)$/);
+
+	    # sort into descending order
+	    return $b_num <=> $a_num;
+	} @$matching_dirs;
+
+	# want sort order to be higest to lowest, for moving 'n' vals up by one
+
+	foreach my $nminus_n (@sorted_matching_dirs) {
+
+	    my $full_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,$nminus_n);
+	    if (-d $full_dir) {
+		# print STDERR "    Increasing by one the file-level document-version history number for:\n";
+		# print STDERR "      $full_dir\n";
+		
+		my ($n) = ($nminus_n =~ m/(\d+)$/);
+		my $new_n = $n + 1;
+
+		my $full_new_dir = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-$new_n");
+		
+		&FileUtils::moveFiles($full_dir,$full_new_dir);
+	    }
+	    else {
+		print STDERR "Warning: skipping $full_dir as it is not a directory\n";
+	    }
+	}
+	
+    }
+    else {
+	my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
+
+	if (!$mkdir_ok) {
+	    print STDERR "Error: DocHistoryFileUtils::prepend_document_version() failed to make directory:\n";
+	    print STDERR "          '$doc_fldv_history_dirname'\n";
+
+	    $status_ok = 0;
+	}
+    }
+
+    if ($status_ok) {
+
+	my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
+
+	my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_doc_dirname],$doc_fldv_history_dirname_nminus1,
+								 { 'strict' => 1, 'exclude_filter_re' => "^$fldv_history_dir\$" } );
+	
+	if (!$hard_link_ok) {
+	    print STDERR "Error: DocHistoryFileUtils::prepend_document_version() failed to hardLink\n";
+	    print STDERR "          '$keepold_doc_dirname' -> '$doc_fldv_history_dirname_nminus1'\n";
+	    
+	    $status_ok = 0;
+	}
+    }
+
+
+    if (!$status_ok) {
+	print STDERR "**** Critical error occurred in creating/updating file-level document-version history\n";
+	print STDERR "**** After determining and correcting the cause of the error, to reset, delete\n";
+	print STDERR "****     your 'archives' folder, and replace it with 'archives_keep'\n";
+	exit 1;
+    }
+}
+
+sub replace_document_version
+{
+    my ($keepold_doc_dirname,$doc_dirname) = @_;
+    
+    my $status_ok = 1;
+    
+    my $doc_fldv_history_dirname = &FileUtils::filenameConcatenate($doc_dirname,$fldv_history_dir);
+    my $doc_fldv_history_dirname_nminus1 = &FileUtils::filenameConcatenate($doc_fldv_history_dirname,"nminus-1");
+    
+    if (!&FileUtils::directoryExists($doc_fldv_history_dirname)) {
+	my $mkdir_ok = &FileUtils::makeDirectory($doc_fldv_history_dirname);
+
+	if (!$mkdir_ok) {
+	    print STDERR "Error: DocHistoryFileUtils::replace_document_version() failed to make directory:\n";
+	    print STDERR "          '$doc_fldv_history_dirname'\n";
+
+	    $status_ok = 0;
+	}
+    }
+    else {
+	# Better to upgrade this method to return a ok_status value
+	&FileUtils::removeFilesRecursive($doc_fldv_history_dirname_nminus1);
+    }	
+	
+    if ($status_ok) {
+
+	my $hard_link_ok = &FileUtils::hardlinkFilesRefRecursive([$keepold_doc_dirname],$doc_fldv_history_dirname_nminus1,
+								 { 'strict' => 1, 'exclude_filter_re' => "^$fldv_history_dir\$" } );	
+	if (!$hard_link_ok) {
+	    print STDERR "Error: DocHistoryFileUtils::replace_document_version() failed to hardLink\n";
+	    print STDERR "          '$keepold_doc_dirname' -> '$doc_fldv_history_dirname_nminus1'\n";
+	    
+	    $status_ok = 0;
+	}
+    }
+
+    if (!$status_ok) {
+	print STDERR "**** Critical error occurred in creating/updating file-level document-version history\n";
+	print STDERR "**** After determining and correcting the cause of the error, to reset, delete\n";
+	print STDERR "****     your 'archives' folder, and replace it with 'archives_keep'\n";
+	exit 1;
+    }
+}
+
+
+
+sub archivedir_keepold_to_archivedir
+{
+    my ($collectcfg, $keepold,$replaceold,$incremental_mode, $archive_info,$archivedir, $archivedir_keepold) = @_;
+
+    my $perform_firsttime_init = 1;
+    my $arcinfo_keepold_doc_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-doc", $archivedir_keepold, $perform_firsttime_init);
+    # my $arcinfo_keepold_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir_keepold, $perform_firsttime_init);
+    
+    # print STDERR "Creating/Updating archives File-Level Document-Version History (fldv-history) directories:\n";
+    my $status_ok = 1;
+    
+    my $archive_keepold_info = new arcinfo($collectcfg->{'infodbtype'});
+    $archive_keepold_info->load_info($arcinfo_keepold_doc_filename);
+    
+    my $archive_keepold_oidinfo_list = $archive_keepold_info->get_OID_list();
+    
+    foreach my $keepold_info_array (@$archive_keepold_oidinfo_list) {
+	
+	my $keepold_OID          = $keepold_info_array->[0];
+	my $keepold_doc_file     = $keepold_info_array->[1];
+	my $keepold_index_status = $keepold_info_array->[2];
+	
+	
+	# Work through all entries in arcinfo_keepold doc-id entries:
+	#
+	#   If keepold entry does *not* exist in (the more up to date) archive_info
+	#   => print out an error.  This should not happen!
+	
+	#   If keepold entry *does* exist in (the more up to date) archive_info
+	#
+	#   (1) if archive_info entry is marked for deletion (D)
+	#      => Do nothing
+	#         This will result in archive_keep and archives versions being wiped
+	#         (respectively after import.pl and buildcol.pl
+	#
+	#   (2) if -keepold on
+	#      => Use hard-linking to turn 'archives_keepold' doc folder
+	#         (without its fldv-history folder) into fldv-history/nminus-1 in the
+	#         the 'archives' doc folder, having first shuffled any existing
+	#         nminus-1, nminus-2 folders down by one value
+	#
+	#   (3) if -replaceold on
+	#      => then a similar process to (2), except the 'archives_keepold'
+	#         doc folder (without its fldv-history folder), directly replaces the
+	#         the 'nminus-1' folder in the 'archives' doc's 'fldv-history' folder
+	
+
+	# Indexing cases to consider:
+	#   I = needs to be Indexed
+	#   R = needs to be Reindexed
+	#   D = needs to be Deleted
+	#   B = has Been indexed
+	#
+	# Specifics of what to do for (2) and (3) dependant at times incremental_mode
+	print STDERR "Document Versioning: Processing $keepold_OID\n";
+	    
+	my $oid_info_array = $archive_info->get_info($keepold_OID);
+	
+	if (defined $oid_info_array) {
+
+	    my $doc_file     = $oid_info_array->[0];
+	    my $index_status = $oid_info_array->[1];
+	    # For completeness, the other position in oid_info_array are
+	    # ->[2] is sort_meta
+	    # ->[3] is group_position
+	    
+	    # **** Need additional work if -groupsize option > 1 is to be supported!
+	    # **** otherwise need to have earlier test to prevent -groupsize > 1 from been used with fldv-history
+	    
+	    if ($index_status eq "D") {
+		#
+		# (1) if archive_info entry marked for deletion (D)
+		#		    
+		print STDERR "  The latest version of $keepold_OID is marked for deletion\n";
+		print STDERR "  => nothing to do right now (will be deleted when buildcol.pl run)\n";		    
+	    }
+	    else {
+		my $keepold_doc_filename = &FileUtils::filenameConcatenate($archivedir_keepold, $keepold_doc_file);
+		my $keepold_doc_dirname  = &util::get_parent_folder($keepold_doc_filename);
+
+		my $doc_filename = &FileUtils::filenameConcatenate($archivedir, $doc_file);
+		my $doc_dirname = &util::get_parent_folder($doc_filename);
+		
+		if ($keepold) {
+		    #
+		    # (2) if -keepold on
+		    #
+
+		    # Still need to consider: I, R, B
+		    # R => always trigger store doc-history
+		    # I => always trigger store doc-history
+		    # B => trigger store doc-history if incremental_mode != "all"
+		    
+		    if ($index_status eq "R") {
+			print STDERR "  Updated version of document from import directory (Index-Status=R)\n";
+			print STDERR "  => keepold: storing snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			prepend_document_version($keepold_doc_dirname,$doc_dirname);
+		    }
+		    elsif ($index_status eq "I") {
+			print STDERR "  Newly generated version of document from 'import/' of existing document from 'archives/' (Index-Status=I)\n";
+			print STDERR "  => keepold: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			prepend_document_version($keepold_doc_dirname,$doc_dirname);
+		    }
+		    elsif ($index_status eq "B") {
+			if ($incremental_mode ne "all") {
+			    print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
+			    print STDERR "  => keepold without incremental: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			    prepend_document_version($keepold_doc_dirname,$doc_dirname);			    
+			}
+			else {
+			    print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
+			    print STDERR "  => no snapshot store needed when in incremental mode\n";
+			}
+		    }
+		}
+		elsif ($replaceold) {
+		    #
+		    # (3) if -replaceold on
+		    #
+
+		    # Still need to consider: I, R, B
+		    # R => always trigger replace nminus-1 doc-history
+		    # I => always trigger replace nminus-1 doc-history
+		    # B => trigger replace nminus-1 doc-history if incremental_mode != "all"
+		    
+		    if ($index_status eq "R") {
+			print STDERR "  Updated version of document from import directory (Index-Status=R)\n";
+			print STDERR "  => replaceold: replacing snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			replace_document_version($keepold_doc_dirname,$doc_dirname);
+		    }
+		    elsif ($index_status eq "I") {
+			print STDERR "  Newly generated version of document from 'import/' of existing document from 'archives/' (Index-Status=I)\n";
+			print STDERR "  => keepold: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			replace_document_version($keepold_doc_dirname,$doc_dirname);
+		    }
+		    elsif ($index_status eq "B") {
+			if ($incremental_mode ne "all") {
+			    print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
+			    print STDERR "  => keepold without incremental: store snapshot of previous version in $fldv_history_dir as 'nminus-1'\n";
+			    replace_document_version($keepold_doc_dirname,$doc_dirname);			    
+			}
+			else {
+			    print STDERR "  Unchanged version of document in 'archives/' (Index-Status=B)\n";
+			    print STDERR "  => no snapshot store needed when in incremental mode\n";
+			}
+		    }
+
+		    
+		}
+		else {
+		    print STDERR "**** Error: Encountered a condition for $keepold_OID that was not expected\n";
+		}
+	    }
+	    
+	}
+	else {
+	    print STDERR "**** Error: Did not find 'archives/' matching document for 'archives_keepold/' OID=$keepold_OID\n";
+	    print STDERR "Skipping!\n";
+	}
+    }
+
+    if ($status_ok) {
+	&FileUtils::removeFilesRecursive($archivedir_keepold);
+    }
+}
+
+1;
