Changeset 23913

Timestamp:

2011-04-16T20:16:19+12:00 (13 years ago)

Author:

ak19

Message:

Additional changes to succeed in more of the OAIServer validation tests: 1. Now passes POST requests introduced in OAI version 2.0. 2. Passes further error tests regarding handling of erroneous parameters for From and/or Until dates and ResumptionToken. More (major) changes yet to be made to pass the final official OAIServer validation test: to do with earliestDateStamp.

Location:

main/trunk/greenstone3/src/java/org/greenstone/gsdl3

Files:

• OAIServer.java (modified) (6 diffs)
• core/OAIReceptionist.java (modified) (10 diffs)
• util/OAIXML.java (modified) (4 diffs)

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/OAIServer.java

@@ -29 +29 @@
 import java.util.Enumeration;
 import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.io.File;
 
@@ -78 +81 @@
   protected String oai_stylesheet = "interfaces/oai/oai2.xsl";
 
+    // there is no getQueryString() method in the HttpServletRequest returned from doPost, 
+    // since that is actually of type apache RequestFacade, and doesn't define such a method
+    protected String queryString = null;
+
   static Logger logger = Logger.getLogger(org.greenstone.gsdl3.OAIServer.class.getName());
   
@@ -223 +230 @@
     String usageInfo = "";
     
+    String query = (queryString == null) ? request.getQueryString() : queryString;
+
     //logged info = general-info + session-info
     usageInfo =
       request.getContextPath()+" "+ //session id
       request.getServletPath()+" "+ //serlvet
-      "["+request.getQueryString()+"]" +" "+ //the query string
+      "["+query+"]" +" "+ //the query string
       "["+usageInfo.trim()+"]" +" "+ // params stored in a session
       request.getRemoteAddr()+" "+   //remote address
@@ -256 +265 @@
     return query.substring(verb_start_index, verb_end_index);
   }
+
   public void doGet(HttpServletRequest request, HttpServletResponse response)
     throws ServletException, IOException {
@@ -276 +286 @@
     //For example, puka.cs.waikato.ac.nz vs www.greenstone.org
     //String base_url = request.getRequestURL().toString();
-    String query = request.getQueryString();
+    // if called by doPost (if this was originally a POST request), var queryString would have been set
+    String query = (queryString == null) ? request.getQueryString() : queryString;
     String[] pairs = (query==null)? null : query.split("&");//split into key/value pairs
     String verb = getVerb(query);
@@ -377 +388 @@
     } 
   }
+
+  // For OAI version 2.0, validation tests indicated that POST needs to be supported. Some
+  // modification was required in order to ensure that the request is passed intact to doGet()
   public void doPost(HttpServletRequest request,
     HttpServletResponse response)
     throws ServletException, IOException {
+
+      // the post method returns a wrapper of type RequestFacade by apache and there
+      // is no getQueryString() method defined for it. Therefore, need to work this out
+      // manually before calling doGet(request, response) so that doGet can work as before.
+      
+      queryString = "";
+      Iterator parameter_entries = request.getParameterMap().entrySet().iterator();
+      while(parameter_entries.hasNext()) {    
+      Map.Entry param_entry = (Map.Entry)parameter_entries.next();
+      String[] paramVals = (String[]) param_entry.getValue();
+      if(paramVals != null) {
+          if(paramVals.length > 0) {
+          logger.error("POST request received: " + param_entry.getKey() + " - " + paramVals[0]);
+          queryString = queryString + "&" + param_entry.getKey() + "=" + paramVals[0];
+          }
+      }   
+      }
+      if(queryString.length() > 0) {
+      queryString = queryString.substring(1);
+      //queryString = java.net.URLEncoder.encode(queryString,"UTF-8");
+      }
+      if(queryString.equals("")) {
+      queryString = null;
+      }
     doGet(request,response);    
   }

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIReceptionist.java

@@ -339 +339 @@
     
     HashMap param_map = OAIXML.getParamMap(params);    
-    if (!isValidParam(param_map, valid_strs) ||
-        !param_map.containsKey(OAIXML.METADATA_PREFIX)) {
-      logger.error("contains invalid params or no metadataPrefix");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    if (!isValidParam(param_map, valid_strs)) {
+    logger.error("One of the params is invalid");
+    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
     } 
+    // param keys are valid, but if there are any date params, check they're of the right format
+    String from = (String)param_map.get(OAIXML.FROM);
+    if(from != null) {  
+    Date from_date = OAIXML.getDate(from);
+    if(from_date == null) {
+        logger.error("invalid date: " + from);
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+    }
+    String until = (String)param_map.get(OAIXML.UNTIL);
+    if(until != null) { 
+    Date until_date = OAIXML.getDate(until);
+    if(until_date == null) {
+        logger.error("invalid date: " + until);
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    } 
+    }    
+    if(from != null && until != null) { // check they are of the same date-time format (granularity)
+    if(from.length() != until.length()) {
+        logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+    }
+
     //ask the message router for a list of oai collections
     NodeList oai_coll = getOAICollectionList();
@@ -350 +373 @@
       logger.info("returned oai collection list is empty");
       return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }
-    //Now that we got a prefix, check and see if it's supported by this repository
-    String prefix_value = (String)param_map.get(OAIXML.METADATA_PREFIX);
-    if (containsMetadataPrefix(prefix_value) == false) {
-      logger.error("requested prefix is not found in OAIConfig.xml");
-      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
     }
     
@@ -390 +407 @@
     }
     
+    // Custom test that expects a metadataPrefix comes here at end so that the official params can
+    // be tested first for errors and their error responses sent off. Required for OAI validation
+    if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
+      logger.error("contains invalid params or no metadataPrefix");
+      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }    
+    
+    //Now that we got a prefix, check and see if it's supported by this repository
+    String prefix_value = (String)param_map.get(OAIXML.METADATA_PREFIX);
+    if (containsMetadataPrefix(prefix_value) == false) {
+      logger.error("requested prefix is not found in OAIConfig.xml");
+      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
+    }
+
     //Now that all validation has been done, I hope, we can send request to the message router
     Element result = null;
@@ -519 +550 @@
     
     HashMap param_map = OAIXML.getParamMap(params);    
-    if (!isValidParam(param_map, valid_strs) ||
-        !param_map.containsKey(OAIXML.METADATA_PREFIX)) {
-      // it must have a metadataPrefix
-      /** Here I disagree with the OAI specification: even if a resumptionToken is 
-       *  included in the request, the metadataPrefix is a must argument. Otherwise
-       *  how would we know what metadataPrefix the harvester requested in his last request?
-       */
-      logger.error("no metadataPrefix");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    } 
+    if (!isValidParam(param_map, valid_strs)) {
+    logger.error("One of the params is invalid");
+    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+    // param keys are valid, but if there are any date params, check they're of the right format
+    String from = (String)param_map.get(OAIXML.FROM);
+    if(from != null) {  
+    Date from_date = OAIXML.getDate(from);
+    if(from_date == null) {
+        logger.error("invalid date: " + from);
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+    }
+    String until = (String)param_map.get(OAIXML.UNTIL);
+    Date until_date = null;
+    if(until != null) { 
+    until_date = OAIXML.getDate(until);
+    if(until_date == null) {
+        logger.error("invalid date: " + until);
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    } 
+    }
+    if(from != null && until != null) { // check they are of the same date-time format (granularity)
+    if(from.length() != until.length()) {
+        logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
+        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+    }
     
     //ask the message router for a list of oai collections
@@ -536 +585 @@
       logger.info("returned oai collection list is empty");
       return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }
-    
-    //Now that we got a prefix, check and see if it's supported by this repository
-    String prefix_value = (String)param_map.get(OAIXML.METADATA_PREFIX);
-    if (containsMetadataPrefix(prefix_value) == false) {
-      logger.error("requested prefix is not found in OAIConfig.xml");
-      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
     }
     
@@ -585 +627 @@
       }
     }
+
+    // Moved the additional custom test that mandates the metadataPrefix here, since official
+    // errors should be caught first, so that their error responses can be sent off first
+    // such that GS2's oaiserver will validate properly.
+    if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
+      // it must have a metadataPrefix
+      /** Here I disagree with the OAI specification: even if a resumptionToken is 
+       *  included in the request, the metadataPrefix is a must argument. Otherwise
+       *  how would we know what metadataPrefix the harvester requested in his last request?
+       */
+      logger.error("no metadataPrefix");
+      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+    }
+        
+    //Now that we got a prefix, check and see if it's supported by this repository
+    String prefix_value = (String)param_map.get(OAIXML.METADATA_PREFIX);
+    if (containsMetadataPrefix(prefix_value) == false) {
+      logger.error("requested prefix is not found in OAIConfig.xml");
+      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
+    }
+
     //Now that all validation has been done, I hope, we can send request to the message router
     Element result = null;
@@ -790 +853 @@
         // get the names
         strs = splitNames(identifier);
+    if(strs == null || strs.length < 3) {
+        logger.error("identifier is not in the form site:coll:id" + identifier);
+        return getMessage(OAIXML.createErrorElement(OAIXML.ID_DOES_NOT_EXIST, ""));
+    }
         String name_of_site = strs[0];
         String coll_name = strs[1];
@@ -851 +918 @@
 
     //do the earliestDatestamp
-    long lastmodified = System.currentTimeMillis();
     //send request to mr to search through the earliest datestamp amongst all oai collections in the repository.
     //ask the message router for a list of oai collections
     NodeList oai_coll = getOAICollectionList();
-    int oai_coll_size = oai_coll.getLength();
-    if (oai_coll_size == 0) {
-      logger.info("returned oai collection list is empty. Set repository earliestDatestamp to be 1970-01-01.");
-      lastmodified = 0;
-    }
-    //the collection build time is determined by the last modified time of the buildConfig.xml file
-    for(int i=0; i<oai_coll_size; i++) {
-      long coll_build_time = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.LASTMODIFIED));
-      lastmodified = (lastmodified > coll_build_time)? coll_build_time : lastmodified;
-    }
-    String earliestDatestamp_str = OAIXML.getTime(lastmodified);
+    long earliestDatestamp = getEarliestDateStamp(oai_coll);
+    String earliestDatestamp_str = OAIXML.getTime(earliestDatestamp);
     Element earliestDatestamp_elem = OAIXML.createElement(OAIXML.EARLIEST_DATESTAMP);
     GSXML.setNodeText(earliestDatestamp_elem, earliestDatestamp_str);
@@ -889 +946 @@
     String [] strs = new String[3];
     int first_colon = identifier.indexOf(":");
+    if(first_colon == -1) {
+    return null;
+    }
     strs[0] = identifier.substring(0, first_colon);
 
@@ -971 +1031 @@
     return converter.nodeToElement(result_node);
   }
+
+    // See OAIConfig.xml
+    // dynamically works out what the earliestDateStamp is, since it varies by collection
+    protected long getEarliestDateStamp(NodeList oai_coll) {
+    //do the earliestDatestamp
+    long lastmodified = System.currentTimeMillis(); 
+    int oai_coll_size = oai_coll.getLength();
+    if (oai_coll_size == 0) {
+        logger.info("returned oai collection list is empty. Setting repository earliestDatestamp to be 1970-01-01.");
+        lastmodified = 0;
+    }
+    //the collection build time is determined by the last modified time of the buildConfig.xml file
+    for(int i=0; i<oai_coll_size; i++) {
+        long coll_build_time = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.LASTMODIFIED));
+        lastmodified = (lastmodified > coll_build_time)? coll_build_time : lastmodified;
+    }
+    return lastmodified;
+    }
 }

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/OAIXML.java

@@ -161 +161 @@
     //stores the date format "yyyy-MM-ddTHH:mm:ssZ"
     public static String granularity = "";
+
+    // http://www.openarchives.org/OAI/openarchivesprotocol.html#DatestampsRequests 
+    // specifies that all repositories must support YYYY-MM-DD (yyyy-MM-dd in Java)
+    // this would be in addition to the other (optional) granularity of above that 
+    // a repository may additionally choose to support.
+    public static final String default_granularity = "yyyy-MM-dd";
+
     //this value is overriden in getOAIConfigXML()
     public static long token_expiration = 7200;
@@ -312 +319 @@
       //initialize the granu_str which might be used by other methods (eg, getDate())
       granularity = GSXML.getNodeText(granu_elem).trim();
+
       //change "yyyy-MM-ddTHH:mm:ssZ" to "yyyy-MM-dd'T'HH:mm:ss'Z'"
       granularity = granularity.replaceAll("T", "'T'");
@@ -462 +470 @@
         date = sdf.parse(pattern);
       } catch(Exception e) {
-        logger.error("invalid date format: " + pattern);
-        return null;
+      if(!default_granularity.equals(granularity)) { // try validating against default granularity
+          try {
+          date = null;
+          sdf = null;
+          sdf = new SimpleDateFormat(default_granularity);
+          date = sdf.parse(pattern);
+          } catch(Exception ex) {
+          logger.error("invalid date format: " + pattern);
+          return null;
+          }
+      } else {
+          logger.error("invalid date format: " + pattern);
+          return null;
+      }
       }
       return date;
@@ -481 +501 @@
         date = sdf.parse(pattern);
       } catch(Exception e) {
-        logger.error("invalid date format: " + pattern);
-        return -1;
+      if(!default_granularity.equals(granularity)) { // try validating against default granularity
+          try {
+          date = null;
+          sdf = null;
+          sdf = new SimpleDateFormat(default_granularity);
+          date = sdf.parse(pattern);
+          } catch(Exception ex) {
+          logger.error("invalid date format: " + pattern);
+          return -1;
+          }
+      } else {
+          logger.error("invalid date format: " + pattern);
+          return -1;
+      }        
       }
       return date.getTime();