Index: /main/trunk/greenstone2/perllib/oaiinfo.pm
===================================================================
--- /main/trunk/greenstone2/perllib/oaiinfo.pm	(revision 31899)
+++ /main/trunk/greenstone2/perllib/oaiinfo.pm	(revision 31900)
@@ -6,4 +6,14 @@
 use constant INFO_DATESTAMP_INDEX => 2;
 
+my $OID_EARLIEST_TIMESTAMP = "earliesttimestamp";
+  # Declaring as my $OID_EARLIEST_TIMESTAMP rather than constant, because it's not straightforward
+  # to use string constant as hash key (need to concat with empty str).
+  # http://perldoc.perl.org/constant.html
+  # But beware of using perl 'constant' as hash key:
+  # https://stackoverflow.com/questions/96848/is-there-any-way-to-use-a-constant-as-hash-key-in-perl
+  # http://forums.devshed.com/perl-programming-6/massive-using-constants-hash-keys-603600.html
+  # https://perlmaven.com/constants-and-read-only-variables-in-perl
+  # http://neilb.org/reviews/constants.html - compares different ways to declare constants in perl
+
 use strict;
 
@@ -16,4 +26,11 @@
 
 # File format read in: OID <tab> (Deletion-)Status <tab> Timestamp <tab> Datestamp
+
+# A special record of the db contains the timestamp of the creation of the oai-inf.db for
+# the collection, representing the collection's earliest datetimestamp.
+# This record has $OID_EARLIEST_TIMESTAMP for OID.
+# Its deletion status is maintained at NA, not applicable.
+# In cases of older oai-inf.db files where there's no earliesttimestamp field, this record
+# is also created but with timestamp set to the oldest lastmodified date in oai-inf.db.
 
 # Deletion status can be:
@@ -21,4 +38,5 @@
 #  D = Doc with OID has been deleted. Timestamp indicates time of deletion
 #  PD = Provisionally Deleted. The associated timestamps are momentarily unaltered.
+#  NA = Not Applicable. Only for the special record with $OID_EARLIEST_TIMESTAMP as OID.
 
 # oaidb is "always incremental": always reflects the I/B/R/D status of archive info db,
@@ -187,6 +205,6 @@
 
 #    print STDERR "@@@@@ oaidb: $self->{'oaidb_file_path'}\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
-    
-    return $do_pd_step;
+
+    return ($do_pd_step, $initdb);
 }
 
@@ -200,18 +218,30 @@
     my ($removeold, $have_manifest) = @_;
 	
-    my $do_pd_step = $self->init_tmpdb($removeold, $have_manifest);
-      # returns 1 if the step to mark oaidb entries as PD is required
+    my ($do_pd_step, $is_new_db) = $self->init_tmpdb($removeold, $have_manifest);
+      # returns 1 for $do_pd_step if the step to mark oaidb entries as PD is required
       # if we're doing full rebuilding and it's NOT the first time creating the oai_inf db, 
       # then the tasks to do with PD (provisionally deleted) OAI OIDs should be carried out
+      # Returns 1 for is_new_db to allow further one time initialisation of the new oai-inf.db
 
     $self->load_info();
     $self->print_info(); # DEBUGGING
 
+    # A special record of the oai-inf.db will contain the timestamp when the oai-inf.db was created.
+    # This represents the collection's "earliest datetimestamp". It should remain unaltered
+    # for as long as oai-inf db exists. This record has the special OID of $OID_EARLIEST_TIMESTAMP.
+    # This record should not be marked as PD, but remain as E, as it can't ever be deleted.
+    # Although the status field for the $OID_EARLIEST_TIMESTAMP record is actually meaningless.    
+    my $save_to_db = $self->insert_coll_earliest_timestamp($is_new_db);    
+    
     if ($do_pd_step) {
 	$self->mark_all_existing_as_provisionallydeleted();
 	$self->print_info(); # DEBUGGING
-
-	# save to db file now that we're done
-	$self->save_info();
+	
+	$save_to_db = 1;	
+    }
+
+    if($save_to_db) {
+	# save changes to $self->{'info'} out to db file, now that we're done
+	$self->save_info(); 
     }
 
@@ -238,5 +268,5 @@
     # all the while ensuring all PDs are changed back to E for OIDs that exist in both arcinfo and oaiinfo db.  
 
-	my $arcinfo_map = $archive_info->{'info'};
+    my $arcinfo_map = $archive_info->{'info'};
 
     foreach my $OID (keys %$arcinfo_map) {
@@ -259,7 +289,7 @@
 	    # an oaicollection. But what if we always maintain an oaidb? Still call $self->index() here.
 	} else {
-		if ($self->{'verbosity'} >= $self->{'verbosity_threshold'}) {
-			print STDERR "### oaiinfo::building_stage_before_indexing(): Unrecognised indexing status $indexing_status\n";
-		}
+	    if ($self->{'verbosity'} >= $self->{'verbosity_threshold'}) {
+		print STDERR "### oaiinfo::building_stage_before_indexing(): Unrecognised indexing status $indexing_status\n";
+	    }
 	}
     }
@@ -297,9 +327,9 @@
 	
     } else {
-		if ($self->{'verbosity'} >= $self->{'verbosity_threshold'}) {
-			print STDERR "@@@@@ In oaiinfo::activate_collection():\n";
-			print STDERR "@@@@@   No tmpdb at $self->{'oaidb_tmp_filepath'}\n";
-			print STDERR "@@@@@   to make 'live' by moving to $self->{'oaidb_live_filepath'}.\n";
-		}
+	if ($self->{'verbosity'} >= $self->{'verbosity_threshold'}) {
+	    print STDERR "@@@@@ In oaiinfo::activate_collection():\n";
+	    print STDERR "@@@@@   No tmpdb at $self->{'oaidb_tmp_filepath'}\n";
+	    print STDERR "@@@@@   to make 'live' by moving to $self->{'oaidb_live_filepath'}.\n";
+	}
     }
 }
@@ -316,5 +346,5 @@
     print STDERR "@@@@@ oaiinfo::mark_all_E_as_PD(): Marking the E entries as PD\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
 
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
 
     foreach my $OID (keys %$infomap) { # Mac Mountain Lion wants %$map, won't accept %$self->{'info'}
@@ -334,5 +364,5 @@
     print STDERR "@@@@@ oaiinfo::mark_all_PD_as_D(): Marking the PD entries as D\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
 
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
 
     foreach my $OID (keys %$infomap) {
@@ -351,12 +381,12 @@
     my $self = shift (@_);
 
-	if ($self->{'verbosity'} < $self->{'verbosity_threshold'}) {
-		return;
-	}	
+    if ($self->{'verbosity'} < $self->{'verbosity_threshold'}) {
+	return;
+    }
 	
     print STDERR "###########################################################\n";
     print STDERR "@@@@@ oaiinfo::print_info(): oaidb in memory contains: \n";
     
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
 
     foreach my $OID (keys %$infomap) {
@@ -369,4 +399,61 @@
 
     print STDERR "###########################################################\n";
+}
+
+
+# When a fresh oai-inf.db is created, this method is called to add the db's special
+# record representing the collection's earliest timestamp.
+# OID=$OID_EARLIEST_TIMESTAMP, deletion_status=NA for not applicable, and current timestamp/date.
+# For older oai-inf.db's that don't yet have this record, a record will be added too,
+# but with the timestamp set to the oldest last modified date for the collection's docs.
+sub insert_coll_earliest_timestamp {
+    my $self = shift (@_);
+    my ($is_new_db) = @_;
+
+    my $current_time = $self->get_current_time();
+    my $save_to_db = 0;
+
+    
+    print STDERR "@@@@@ oaiinfo::insert_coll_earliest_timestamp(): " if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
+    
+    if($is_new_db) {
+	
+	print STDERR "New db. Setting timestamp of oai-inf.db creation.\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
+	
+	$self->set_info($OID_EARLIEST_TIMESTAMP, "NA", $current_time);
+	$save_to_db = 1;
+    }
+    
+    else { # oai-inf.db already exists, ensure it has an [$OID_EARLIEST_TIMESTAMP] set
+
+	my $earliesttimestamp_record = $self->{'info'}->{$OID_EARLIEST_TIMESTAMP};
+	
+	if (!defined $earliesttimestamp_record) {
+	    # oai-inf.db exists, but doesn't contain an [$OID_EARLIEST_TIMESTAMP] record yet.
+	    # Let's create one for it:
+	    # Work out the earliest lastmodified datetime in the collection, by inspecting
+	    # the last modified timestamp for each doc in the collection
+	    
+	    my $earliest_timestamp = $current_time;
+	    
+	    my $infomap = $self->{'info'}; # Mac Mountain Lion wants %$map, won't accept %$self->{'info'}	    
+	    foreach my $OID (keys %$infomap) {
+		my $OID_info = $self->{'info'}->{$OID};
+		my $lastmodified = $OID_info->[INFO_TIMESTAMP_INDEX];
+		if($lastmodified < $earliest_timestamp) {
+		    $earliest_timestamp = $lastmodified;
+		}
+	    }
+	    
+	    print STDERR "Collection timestamp not yet set for $OID_EARLIEST_TIMESTAMP. Setting to earliest found: $earliest_timestamp\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
+	    
+	    $self->set_info($OID_EARLIEST_TIMESTAMP, "NA", $earliest_timestamp);
+	    $save_to_db = 1;
+	} else {
+	    print STDERR "Collection timestamp was already set\n" if $self->{'verbosity'} >= $self->{'verbosity_threshold'};
+	}
+    }
+
+    return $save_to_db;
 }
 
@@ -422,5 +509,5 @@
 
     # the following method will set to current time if no timestamp provided,
-    # But by explicit here, the code is easier to follow
+    # But by being explicit here, the code is easier to follow
     $self->set_info($OID, "D", $self->get_current_time());
 
@@ -540,8 +627,8 @@
 
     # write out again. Open file for overwriting, not appending.
-    # Then write out data structure $self->{'info'} that's been maintaining the data in-memory. 
+    # Then write out data structure $self->{'info'} that has been maintaining the data in-memory. 
     my $infodb_handle = &dbutil::open_infodb_write_handle($infodbtype, $filename);
 
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
     foreach my $oid ( keys %$infomap ) {
 	my $OID_info = $self->{'info'}->{$oid};
@@ -599,5 +686,5 @@
     my @list = ();
 	
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
     foreach my $OID (keys %$infomap) {	
 	my $OID_info = $self->{'info'}->{$OID};
@@ -617,5 +704,5 @@
 sub size {
     my $self = shift (@_);
-	my $infomap = $self->{'info'};
+    my $infomap = $self->{'info'};
     return (scalar keys %$infomap);
 }
