Changeset 31230

Timestamp:

2016-12-13T20:36:01+13:00 (7 years ago)

Author:

ak19

Message:

Commit for GS3 server side part of OAI deletion police implementation. Still to implement the GS2 server side part. The earlier commits implemented the PERL side, the oai-inf db implementation. I think I've now got the GS3 server side working, but have yet to try validating against the OAI validator. (I need to test that on a machine that is publicly accessible).

Location:

main/trunk/greenstone3/src/java/org/greenstone/gsdl3

Files:

• service/OAIPMH.java (modified) (12 diffs)
• util/FlatDatabaseWrapper.java (modified) (2 diffs)
• util/GDBMWrapper.java (modified) (2 diffs)
• util/GSFile.java (modified) (1 diff)
• util/JDBMWrapper.java (modified) (2 diffs)
• util/OAIXML.java (modified) (1 diff)
• util/SimpleCollectionDatabase.java (modified) (2 diffs)

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/service/OAIPMH.java

@@ -58 +58 @@
   
   protected SimpleCollectionDatabase coll_db = null;
+  protected SimpleCollectionDatabase oaiinf_db = null;
   
   protected String site_name = "";
@@ -76 +77 @@
     super.cleanUp();//??
     this.coll_db.closeDatabase();
+    this.oaiinf_db.closeDatabase();
   }
   /** configure this service 
@@ -114 +116 @@
 
     if (index_stem == null || index_stem.equals("")) {
-      index_stem = this.cluster_name;
+    index_stem = this.cluster_name; // index_stem is the name of the db in indext/text, it is <colname>.<db>
     }
     if (infodb_type == null || infodb_type.equals("")) {
@@ -125 +127 @@
       return false;
     }
-    
-    // Open database for querying
+
+    oaiinf_db = new SimpleCollectionDatabase(infodb_type);
+    if (!oaiinf_db.databaseOK()) {
+      logger.error("Couldn't create the oai-inf database of type "+infodb_type);
+      oaiinf_db = null;
+      return false;
+    }
+
+    
+    // Open databases for querying
     String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, index_stem, infodb_type);
     if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ)) {
       logger.error("Could not open collection database!");
       return false;
+    }
+    // the oaiinf_db is called oai-inf.<infodb_type_extension>
+    String oaiinf_db_file = GSFile.OAIInfoDatabaseFile(this.site_home, this.cluster_name, "oai-inf", infodb_type);
+    if (oaiinf_db != null && !this.oaiinf_db.openDatabase(oaiinf_db_file, SimpleCollectionDatabase.READ)) {
+      logger.warn("Could not open oai-inf database for collection + " + this.cluster_name + "!");
     }
     
@@ -332 +347 @@
       return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
     }
+
+    Document doc = XMLConverter.newDOM();
     
     String oid = param_map.get(OAIXML.OID); // TODO should this be identifier???
+    boolean OID_is_deleted = false;
+    long millis = -1;
+
+    DBInfo oai_info = null;
+    if(oaiinf_db != null) {
+    oai_info = this.oaiinf_db.getInfo(oid);
+    if (oai_info == null) {
+        logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
+    } else  {
+        String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
+        if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
+        OID_is_deleted = true;
+
+        // get the right timestamp for deletion: from oaiinf db
+        String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // in seconds presumably, like oailastmodified in the collection index db        
+        
+        millis = Long.parseLong(timestamp)*1000; // in milliseconds
+        }
+    }
+    }
 
     //get a DBInfo object of the identifier; if this identifier is not present in the database,
@@ -339 +376 @@
     DBInfo info = this.coll_db.getInfo(oid);
     if (info == null) {
-      logger.error("OID: " + oid + " is not present in the database.");
-      return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
-    }
-
-    Document doc = XMLConverter.newDOM();
-    ArrayList<String> keys = new ArrayList<String>(info.getKeys());
-    long millis = getDateStampMillis(info);
-    String oailastmodified = ""; 
-    if (millis != -1) {
-      oailastmodified = OAIXML.getTime(millis);
-    }
+      logger.error("OID: " + oid + " is not present in the collection database.");
+      //return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, ""); // may exist as deleted in oai-inf db
+    }
+    else if (millis == -1) { // so !OID_is_deleted, get oailastmodified from collection's index db
+    ArrayList<String> keys = new ArrayList<String>(info.getKeys()); 
+    millis = getDateStampMillis(info);  
+    }
+    String oailastmodified = (millis == -1) ? "" : OAIXML.getTime(millis);
+    
 
     Element get_record_response = doc.createElement(GSXML.RESPONSE_ELEM);
@@ -356 +391 @@
     Element record = doc.createElement(OAIXML.RECORD);
     //compose the header element
-    record.appendChild(createHeaderElement(doc, oid, oailastmodified));      
-    //compose the metadata element
-    record.appendChild(createMetadataElement(doc, prefix, info));
+    record.appendChild(createHeaderElement(doc, oid, oailastmodified, OID_is_deleted));      
+    if(!OID_is_deleted) {
+    //compose the metadata element
+    record.appendChild(createMetadataElement(doc, prefix, info));
+    }
     get_record.appendChild(record);
     return get_record_response;
@@ -421 +458 @@
       return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
     }
-    ArrayList<String> oid_list = getChildrenIds(OAIXML.BROWSELIST);
+
+    // get list of oids
+    ArrayList<String> oid_list = null;
+    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
+    oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
+    
+    if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
+        logger.warn("@@@@@@@@@@@@@ NO OIDs in oai-inf db for " + this.cluster_name);
+        oid_list = getChildrenIds(OAIXML.BROWSELIST);
+    }
+    }
+
     if (oid_list == null) {
-      logger.error("No matched records found in collection: browselist is empty");
+      logger.error("No matched records found in collection: oai-inf and index db's browselist are empty");
       return OAIXML.createErrorResponse(OAIXML.NO_RECORDS_MATCH, "");
     }
@@ -438 +486 @@
     for(int i=0; i<oid_list.size(); i++) {
       String oid = oid_list.get(i);
+      boolean OID_is_deleted = false;
+      long millis = -1;
+
+      DBInfo oai_info = null;
+      if(oaiinf_db != null) {
+      oai_info = this.oaiinf_db.getInfo(oid);
+      if (oai_info == null) {
+          logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
+      } else  {
+          String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
+          if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
+          OID_is_deleted = true;
+          
+          // get the right timestamp for deletion: from oaiinf db
+          String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // in seconds presumably, like oailastmodified in the collection index db      
+          
+          millis = Long.parseLong(timestamp)*1000; // in milliseconds
+          }
+      }
+      }
       DBInfo info = this.coll_db.getInfo(oid);
-      if (info == null) {
-        logger.error("Database does not contains information about oid: " +oid);
-        continue;
-      }
-      
-      long millis = getDateStampMillis(info);
+      if (info == null) { // can happen if oid was deleted, in which case only oai_info keeps a record of it
+        logger.error("Collection database does not contain information about oid: " +oid);
+      }
+      else if (millis == -1) { // so !OID_is_deleted, get oailastmodified from collection's index db
+      
+      millis = getDateStampMillis(info);
+      }
+
       Date this_date = null;
       if (millis == -1) {
-    if (from_date != null || until_date !=null) {
-      continue; // if this doc doesn't have a date for some reason, and
-      // we are doing a date range, then don't include it.
-    }
+      if (from_date != null || until_date !=null) {
+          continue; // if this doc doesn't have a date for some reason, and
+          // we are doing a date range, then don't include it.
+      }
       } else {
-    this_date = new Date(millis);
-    if (from_date != null) {
-      if(this_date.before(from_date)) {
-        continue;
+      this_date = new Date(millis);
+      if (from_date != null) {
+          if(this_date.before(from_date)) {
+          continue;
+          }
       }
-    }
-    if (until_date != null) {
-      if (this_date.after(until_date)) {
-        continue;
-      }
-    }    
-      }  
- 
+      if (until_date != null) {
+          if (this_date.after(until_date)) {
+          continue;
+          }
+      }    
+      }
+      
+      
+      // compose a record for adding header and metadata
+      Element record = doc.createElement(OAIXML.RECORD);
+      list_items.appendChild(record);
+      //compose the header element
+      record.appendChild(createHeaderElement(doc, oid, OAIXML.getTime(millis), OID_is_deleted));
+
+
       //Now check that this id has metadata for the required prefix.
-      if (documentContainsMetadata(info, set_of_elems)) {
-    // YES, it does have some metadata for this prefix
-    if (include_metadata) {
-      // compose a record and add header and metadata
-      Element record = doc.createElement(OAIXML.RECORD);
-      list_items.appendChild(record);
-      //compose the header element
-      record.appendChild(createHeaderElement(doc, oid, OAIXML.getTime(millis)));      
-      //compose the metadata element
-      record.appendChild(createMetadataElement(doc, prefix, info));
-    } else {
-      //compose the header element and append it
-      list_items.appendChild(createHeaderElement(doc, oid, OAIXML.getTime(millis)));      
-    }
-      } // otherwise we won't include this oid.
+      if (info != null && documentContainsMetadata(info, set_of_elems)) {
+      // YES, it does have some metadata for this prefix        
+      
+        if (include_metadata) {     
+        //compose the metadata element
+        record.appendChild(createMetadataElement(doc, prefix, info));
+        } /*else {
+          //compose the header element and append it
+          list_items.appendChild(createHeaderElement(doc, oid, OAIXML.getTime(millis)));      
+          }*/
+      } // otherwise we won't include this oid. with meta
+      
+      
+      
     }//end of for(int i=0; i<oid_list.size(); i++) of doing thru each record
     
@@ -620 +697 @@
 
   /** create a header element used when processing requests like ListRecords/GetRecord/ListIdentifiers
-   */
-  protected Element createHeaderElement(Document doc, String oid, String oailastmodified) {    
+   */  
+  protected Element createHeaderElement(Document doc, String oid, String oailastmodified, boolean deleted) {
+
         Element header = doc.createElement(OAIXML.HEADER);
+    
+    // if deleted, get the date and change oailastmodified to timestamp in oaiinfo
+    if(deleted) {
+        header.setAttribute(OAIXML.OAI_INF_STATUS, OAIXML.HEADER_STATUS_ATTR_DELETED); // set the header status to deleted
+        // then the timestamp for deletion will be from oai-inf database 
+    }
+    
         Element identifier = doc.createElement(OAIXML.IDENTIFIER);
     GSXML.setNodeText(identifier, coll_name + ":" + oid);
@@ -648 +733 @@
       return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
     }
+
+    /*
+    ArrayList<String> oid_list = null;
+    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
+    oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
+    
+    if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
+        oid_list = getChildrenIds(OAIXML.BROWSELIST);
+    }
+    }
+    */
+    // assume meta formats are only for OIDs that have not been deleted
+    // so don't need to check oai-inf db, and can just check collection's index db for list of OIDs
     ArrayList<String> oid_list = getChildrenIds(OAIXML.BROWSELIST);
     if (oid_list == null || oid_list.contains(oid) == false) {
@@ -724 +822 @@
    * The name may be in the form: <name>,<mapped name>, in which the mapped name is
    * optional. The mapped name is looked up in the DBInfo; if not present, use the first
-   * name which is mendatory.
+   * name which is mandatory.
    */
   protected boolean containsMetadata(DBInfo info, String[] metadata_names) {

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/FlatDatabaseWrapper.java

@@ -18 +18 @@
  */
 package org.greenstone.gsdl3.util;
+
+import java.util.ArrayList;
 
 public interface FlatDatabaseWrapper {
@@ -44 +46 @@
   public boolean deleteKey(String key);
 
+  /** Returns all the keys of the database as String */
+  public ArrayList<String> getAllEntryKeys();
+
   /** returns a string of key-value entries that can be 
    *    printed for debugging purposes*/

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/GDBMWrapper.java

@@ -27 +27 @@
 import java.io.UnsupportedEncodingException;
 import java.io.File;
+import java.util.ArrayList;
 
 /**
@@ -262 +263 @@
     }
 
+    /** Returns all the keys of the database as Strings */
+    public ArrayList<String> getAllEntryKeys() {
+
+    ArrayList<String> keys = new ArrayList<String>();
+    
+    try {
+        java.util.Enumeration e = db_.keys();
+        while (e.hasMoreElements()) {
+        
+        Object key = e.nextElement();
+        keys.add((String)key);
+        
+        }    
+    } catch (Exception e) {
+        logger.error("Exception encountered when trying to GDBMWrapper.getAllEntries():" + e);
+    }
+    
+    return keys;
+    }
+    
+
     /**
      * returns a string of key-value entries that can be printed for debugging

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/GSFile.java

@@ -372 +372 @@
     return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "index" + File.separatorChar + "text" + File.separatorChar + index_stem + db_ext;
     }
+    
+    /** the oai-inf database file in the collection's etc folder */
+    static public String OAIInfoDatabaseFile(String site_home, String collection_name, String db_tailname, String database_type)
+    {
+    String db_ext = DBHelper.getDBExtFromDBType(database_type);
+    if (null == db_ext || db_ext.equals("")) {
+        logger.warn("Could not recognise database type \"" + database_type + "\", defaulting to GDBM and extension \".gdb\"");
+        // assume gdbm
+        db_ext = ".gdb";
+    }
+    return site_home + File.separatorChar + "collect" + File.separatorChar + collection_name + File.separatorChar + "etc" + File.separatorChar + db_tailname + db_ext; // db tailname should be oai-inf
+    }
+
 
     /** the archives database file - */

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/JDBMWrapper.java

@@ -217 +217 @@
     }
 
+    /** Returns all the keys of the database as Strings */
+    public ArrayList<String> getAllEntryKeys() {
+    
+    ArrayList<String> keys = new ArrayList<String>();
+    
+    try {
+        FastIterator iter = hashtable_.keys();
+        
+        String key = (String) iter.next();
+        
+        while (key != null) {
+        keys.add(key);
+        key = (String) iter.next();
+        }
+        
+        recman_.commit();
+    } catch (IOException e) {
+        logger.error("Failed to get all keys from JDBM database");
+        return null;
+    }
+    
+    return keys;
+    }
+    
+
     /**
      * returns a string of key-value entries that can be printed for debugging
@@ -247 +272 @@
         catch (IOException e)
         {
-            logger.error("Failed get all keys and values from JDBM database");
+            logger.error("Failed to get all keys and values from JDBM database");
             return null;
         }

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/OAIXML.java

@@ -147 +147 @@
    public static final String OAI_SERVICE_UNAVAILABLE = "OAI service unavailable";
    public static final String OID = "OID";
-    
+
+    // The node id in the oai-inf database of the collection, which should contain all the OIDs in the db
+    public static final String OAI_INF_STATUS = "status"; // D = Deleted; E = Existing (PD = Provisionally Deleted but PD should not be present at this stage in the oai-inf database)
+    public static final String OAI_INF_TIMESTAMP = "timestamp"; // the time of deletion/last (re)indexing of doc
+    public static final String OAI_INF_DATESTAMP = "datestamp"; // date version of timestamp    
+    // oai-inf db values for OAI_INF_STATUS
+    public static final String OAI_INF_DELETED = "D";
+    public static final String OAI_INF_EXISTS = "E";
+    public static final String OAI_INF_PROVISIONALLY_DELETED = "PD";
+    // header values for ListRecords/GetRecord and listIdentifiers
+    public static final String HEADER_STATUS_ATTR_DELETED = "deleted";
+
   public static final String OAI_SERVICE_RACK = "OAIPMH";
   //system-dependent file separator, maybe '/' or '\'

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/SimpleCollectionDatabase.java

@@ -19 +19 @@
 package org.greenstone.gsdl3.util;
 
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Set;
@@ -216 +217 @@
     }
 
+    public ArrayList<String> getAllKeys() {
+    return this.coll_db.getAllEntryKeys();
+    }
+
     /**
      * converts a greenstone OID to an internal docnum, returning a Long -