Index: /main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIReceptionist.java
===================================================================
--- /main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIReceptionist.java	(revision 28856)
+++ /main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIReceptionist.java	(revision 28857)
@@ -1,2 +1,21 @@
+/*
+ *    OAIReceptionist.java
+ *    Copyright (C) 2012 New Zealand Digital Library, http://www.nzdl.org
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
 package org.greenstone.gsdl3.core;
 
@@ -28,10 +47,5 @@
   /** The unique  repository identifier */
   protected String repository_id = null;
-  
-  /** container Document to create XML Nodes for requests sent to message router
-   *  Not used for response 
-   */
-  protected Document doc=null;
-  
+    
   /** a converter class to parse XML and create Docs */
   protected XMLConverter converter=null;
@@ -46,5 +60,4 @@
   protected ModuleInterface mr = null;
   
-
   // Some of the data/responses will not change while the servlet is running, so
   // we can cache them
@@ -52,12 +65,24 @@
   /** A list of all the collections available to this OAI server */
   protected NodeList collection_list = null;
-
+  /** a vector of the names, for convenience */
+  protected Vector<String> collection_name_list = null;
+  /** If this is true, then there are no OAI enabled collections, so can always return noRecordsMatch (after validating the request params) */
+  protected boolean noRecordsMatch = false;
+      
+  /** A set of all known 'sets' */
+  protected HashSet<String> set_set = null;
+
+  protected boolean has_super_colls = false;
+  /** a hash of super set-> collection list */
+  protected HashMap<String, Vector<String>> super_coll_map = null;
   /** The identify response */
   protected Element identify_response = null;
-  
+  /** The list set response */
+  protected Element listsets_response = null;
+  /** the list metadata formats response */
+  protected Element listmetadataformats_response = null;
+
   public OAIReceptionist() {
     this.converter = new XMLConverter();
-    this.doc = this.converter.newDOM();
-
   }
   
@@ -87,13 +112,152 @@
     resume_after = getResumeAfter();
     
-    repository_id = getRepositoryId(); 
-    collection_list = getOAICollectionList();
+    repository_id = getRepositoryIdentifier(); 
+    if (!configureSetInfo()) {
+      // there are no sets
+      logger.error("No sets (collections) available for OAI");
+      return false;
+    }
 
     //clear out expired resumption tokens stored in OAIResumptionToken.xml
-    OAIXML.init();
-    OAIXML.clearExpiredTokens();
+    OAIResumptionToken.init();
+    OAIResumptionToken.clearExpiredTokens();
     
     return true;
   }
+
+  // assuming that sets are static. If collections change then the servlet 
+  // should be restarted.
+  private boolean configureSetInfo() {
+    // do we have any super colls listed in web/WEB-INF/classes/OAIConfig.xml?
+    // Will be like 
+    // <oaiSuperSet>
+    //   <SetSpec>xxx</SetSpec>
+    //   <setName>xxx</SetName>
+    //   <SetDescription>xxx</setDescription>
+    // </oaiSuperSet>
+    // The super set is listed in OAIConfig, and collections themselves state
+    // whether they are part of the super set or not.
+    NodeList super_coll_list = this.oai_config.getElementsByTagName(OAIXML.OAI_SUPER_SET);
+    HashMap<String, Element> super_coll_data = new HashMap<String, Element>();
+    if (super_coll_list.getLength() > 0) {
+      this.has_super_colls = true;
+      for (int i=0; i<super_coll_list.getLength(); i++) {
+	Element super_coll = (Element)super_coll_list.item(i);
+	Element set_spec = (Element)GSXML.getChildByTagName(super_coll, OAIXML.SET_SPEC);
+	if (set_spec != null) {
+	  String name = GSXML.getNodeText(set_spec);
+	  if (!name.equals("")) {
+	    super_coll_data.put(name, super_coll);
+	    logger.error("adding in super coll "+name);
+	  }
+	}
+      }
+    
+      if (super_coll_data.size()==0) {
+	this.has_super_colls = false;
+      }
+    }
+    if (this.has_super_colls == true) {
+      this.super_coll_map = new HashMap<String, Vector<String>>();
+    }
+    this.set_set = new HashSet<String>();
+
+    // next, we get a list of all the OAI enabled collections
+    // We get this by sending a listSets request to the MR
+    Document doc = this.converter.newDOM();
+    Element message = doc.createElement(GSXML.MESSAGE_ELEM);
+    
+    Element request = GSXML.createBasicRequest(doc, OAIXML.OAI_SET_LIST, "", null);
+    message.appendChild(request);
+    Node msg_node = mr.process(message);
+    
+    if (msg_node == null) {
+      logger.error("returned msg_node from mr is null");
+      return false;
+    }
+    Element resp = (Element)GSXML.getChildByTagName(msg_node, GSXML.RESPONSE_ELEM);
+    Element coll_list = (Element)GSXML.getChildByTagName(resp, GSXML.COLLECTION_ELEM + GSXML.LIST_MODIFIER);
+    if (coll_list == null) {
+      logger.error("coll_list is null");
+      return false;
+    }
+
+    NodeList list = coll_list.getElementsByTagName(GSXML.COLLECTION_ELEM);
+    int length = list.getLength();
+    if (length == 0) {
+      logger.error("length is 0");
+      noRecordsMatch = true;
+      return false;
+    }
+
+    this.collection_list = list;
+    this.collection_name_list = new Vector<String>();
+    
+    Document listsets_doc = this.converter.newDOM();
+    Element listsets_element = listsets_doc.createElement(OAIXML.LIST_SETS);
+    this.listsets_response = getMessage(listsets_doc, listsets_element);
+    
+    // Now, for each collection, get a list of all its sets
+    // might include subsets (classifiers) or super colls
+    // We'll reuse the first message, changing its type and to atts
+    request.setAttribute(GSXML.TYPE_ATT, "");
+    StringBuffer to = new StringBuffer();
+    for (int i=0; i<collection_list.getLength(); i++) {
+      if (i!=0) {
+	to.append(',');
+      }
+      String coll_id =((Element) collection_list.item(i)).getAttribute(GSXML.NAME_ATT);
+      logger.error("coll_id = "+coll_id);
+      to.append(coll_id+"/"+OAIXML.LIST_SETS);
+      this.collection_name_list.add(coll_id);
+    }
+    logger.error ("to att = "+to.toString());
+    request.setAttribute(GSXML.TO_ATT, to.toString());
+    // send to MR
+    msg_node = mr.process(message);
+    logger.error(this.converter.getPrettyString(msg_node));
+    NodeList response_list =  ((Element)msg_node).getElementsByTagName(GSXML.RESPONSE_ELEM);
+    for (int c=0; c<response_list.getLength(); c++) {
+      // for each collection's response
+      Element response = (Element)response_list.item(c);
+      String coll_name = GSPath.getFirstLink(response.getAttribute(GSXML.FROM_ATT));
+      logger.error("coll from response "+coll_name);
+      NodeList set_list = response.getElementsByTagName(OAIXML.SET);
+      for (int j=0; j<set_list.getLength(); j++) {
+	// now check if it a super collection
+	Element set = (Element)set_list.item(j);
+	String set_spec = GSXML.getNodeText((Element)GSXML.getChildByTagName(set, OAIXML.SET_SPEC));
+	logger.error("set spec = "+set_spec);
+	// this may change if we add site name back in
+	// setSpecs will be collname or collname:subset or supercollname
+	if (set_spec.indexOf(":")==-1 && ! set_spec.equals(coll_name)) {
+	  // it must be a super coll spec
+	  logger.error("found super coll, "+set_spec);
+	  // check that it is a valid one from config
+	  if (this.has_super_colls == true && super_coll_data.containsKey(set_spec)) {
+	    Vector <String> subcolls = this.super_coll_map.get(set_spec);
+	    if (subcolls == null) {
+	      logger.error("its new!!");
+	      // not in there yet
+	      subcolls = new Vector<String>();
+	      this.set_set.add(set_spec);
+	      this.super_coll_map.put(set_spec, subcolls);
+	      // the first time a supercoll is mentioned, add into the set list
+	      logger.error("finding the set info "+this.converter.getPrettyString(super_coll_data.get(set_spec)));
+	      listsets_element.appendChild(GSXML.duplicateWithNewName(listsets_doc, super_coll_data.get(set_spec), OAIXML.SET, true));
+	    }
+	    // add this collection to the list for the super coll
+	    subcolls.add(coll_name);
+	  }
+	} else { // its either the coll itself or a subcoll
+	  // add in the set
+	  listsets_element.appendChild(listsets_doc.importNode(set, true));
+	  this.set_set.add(set_spec);
+	}
+      } // for each set in the collection
+    } // for each OAI enabled collection
+    return true;
+  }
+
   /** process using strings - just calls process using Elements */
   public String process(String xml_in) {
@@ -104,11 +268,14 @@
   }
 
-  //Compose a message element used to send back to the OAIServer servlet. 
+  //Compose a message/response element used to send back to the OAIServer servlet. 
   //This method is only used within OAIReceptionist
-  private Element getMessage(Element e) {
-    Element msg = OAIXML.createElement(OAIXML.MESSAGE);
-    msg.appendChild(OAIXML.getResponse(e));
+  private Element getMessage(Document doc, Element e) {
+    Element msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element response = doc.createElement(GSXML.RESPONSE_ELEM);
+    msg.appendChild(response);
+    response.appendChild(e);
     return msg;
   }
+
   /** process - produce xml data in response to a request
    * if something goes wrong, it returns null -
@@ -123,5 +290,5 @@
     if (!message.getTagName().equals(GSXML.MESSAGE_ELEM)) {
       logger.error(" Invalid message. GSDL message should start with <"+GSXML.MESSAGE_ELEM+">, instead it starts with:"+message.getTagName()+".");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "Internal messaging error");
     }
    
@@ -130,5 +297,5 @@
     if (request == null) {
       logger.error(" message had no request!");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "Internal messaging error");
     }
     //At this stage, the value of 'to' attribute of the request must be the 'verb'
@@ -139,66 +306,26 @@
     }
     if (verb.equals(OAIXML.LIST_METADATA_FORMATS)) {
-      return doListMetadataFormats(message);
+      return doListMetadataFormats(request);
     }
     if (verb.equals(OAIXML.LIST_SETS)) {
-      return doListSets(message);
+      // we have composed the list sets response on init
+      // Note this means that list sets never uses resumption tokens
+      return this.listsets_response; 
     }
     if (verb.equals(OAIXML.GET_RECORD)) {
-      return doGetRecord(message);
+      return doGetRecord(request);
     }
     if (verb.equals(OAIXML.LIST_IDENTIFIERS)) {
-      return doListIdentifiers(message);
+      return doListIdentifiersOrRecords(request,OAIXML.LIST_IDENTIFIERS , OAIXML.HEADER);
     }
     if (verb.equals(OAIXML.LIST_RECORDS)) {
-      return doListRecords(message);
-    }
-    return getMessage(OAIXML.createErrorElement("Unexpected things happened", ""));
-    
-  }
-  /** send a request to the message router asking for a list of collections that support oai
-   *  The type attribute must be changed from 'oaiService' to 'oaiSetList'
-   */
-  private NodeList getOAICollectionList() {
-    Element message = this.doc.createElement(OAIXML.MESSAGE);
-    Element request = this.doc.createElement(OAIXML.REQUEST);
-    message.appendChild(request);
-    request.setAttribute(OAIXML.TYPE, OAIXML.OAI_SET_LIST);
-    request.setAttribute(OAIXML.TO, "");
-    Node msg_node = mr.process(message);
-    
-    if (msg_node == null) {
-      logger.error("returned msg_node from mr is null");
-      return null;
-    }
-    Element resp = (Element)GSXML.getChildByTagName(msg_node, OAIXML.RESPONSE);
-    Element coll_list = (Element)GSXML.getChildByTagName(resp, OAIXML.COLLECTION_LIST);
-    if (coll_list == null) {
-      logger.error("coll_list is null");
-      return null;
-    }
-    //logger.info(GSXML.xmlNodeToString(coll_list));
-    NodeList list = coll_list.getElementsByTagName(OAIXML.COLLECTION);
-    int length = list.getLength();
-    if (length == 0) {
-      logger.error("length is 0");
-      return null;
-    }
-    return list;
-  }
-  /**Exclusively called by doListSets()*/
-  private void getSets(Element list_sets_elem, NodeList oai_coll, int start_point, int end_point) {
-    for (int i=start_point; i<end_point; i++) {
-      String coll_spec = ((Element)oai_coll.item(i)).getAttribute(OAIXML.NAME);
-      String coll_name = coll_spec.substring(coll_spec.indexOf(":") + 1);
-      Element set = OAIXML.createElement(OAIXML.SET);
-      Element set_spec = OAIXML.createElement(OAIXML.SET_SPEC);
-      GSXML.setNodeText(set_spec, coll_spec);
-      set.appendChild(set_spec);
-      Element set_name = OAIXML.createElement(OAIXML.SET_NAME);
-      GSXML.setNodeText(set_name, coll_name);
-      set.appendChild(set_name);
-      list_sets_elem.appendChild(set);
-    }
-  }
+      return doListIdentifiersOrRecords(request, OAIXML.LIST_RECORDS, OAIXML.RECORD);
+    }
+    // should never get here as verbs were checked in OAIServer
+    return OAIXML.createErrorMessage(OAIXML.BAD_VERB, "Unexpected things happened");
+    
+  }
+
+
   private int getResumeAfter() {
     Element resume_after = (Element)GSXML.getChildByTagName(oai_config, OAIXML.RESUME_AFTER);
@@ -206,6 +333,6 @@
     return -1;
   }
-  private String getRepositoryId() {
-    Element ri = (Element)GSXML.getChildByTagName(oai_config, OAIXML.REPOSITORY_ID);
+  private String getRepositoryIdentifier() {
+    Element ri = (Element)GSXML.getChildByTagName(oai_config, OAIXML.REPOSITORY_IDENTIFIER);
     if (ri != null) { 
       return GSXML.getNodeText(ri);
@@ -213,136 +340,13 @@
     return "";
   }
-  /** method to compose a set element
-   */
-  private Element doListSets(Element msg){
-    logger.info("");
-    // option: resumptionToken
-    // exceptions: badArgument, badResumptionToken, noSetHierarchy
-    Element list_sets_elem = OAIXML.createElement(OAIXML.LIST_SETS);
-
-    int oai_coll_size = collection_list.getLength(); 
-    if (oai_coll_size == 0) {
-      return getMessage(list_sets_elem);
-    }
-    
-    Element req = (Element)GSXML.getChildByTagName(msg, GSXML.REQUEST_ELEM);
-    if (req == null) {
-      logger.error("req is null");
-      return null;
-    }
-    //params list only contains the parameters other than the verb
-    NodeList params = GSXML.getChildrenByTagName(req, OAIXML.PARAM);
-    Element param = null;
-    int smaller = (oai_coll_size>resume_after)? resume_after : oai_coll_size;
-    if (params.getLength() > 1) {
-      //Bad argument. Can't be more than one parameters for ListMetadataFormats verb
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    }
-    if(params.getLength() == 0) {
-      //this is requesting a list of sets in the whole repository
-      /** there is no resumeptionToken in the request, we check whether we need
-       *  to send out resumeptionToken by comparing the total number of sets in this
-       *  repository and the specified value of resumeAfter
-       */
-      if(resume_after < 0 || oai_coll_size <= resume_after) {
-        //send the whole list of records
-        //all data are sent on the first request. Therefore there should be
-        //no resumeptionToken stored in OAIConfig.xml.
-        //As long as the verb is 'ListSets', we ignore the rest of the parameters
-        getSets(list_sets_elem, collection_list, 0, oai_coll_size);
-        return getMessage(list_sets_elem);
-      }
-      
-      //append required sets to list_sets_elem (may be a complete or incomplete list)
-      getSets(list_sets_elem, collection_list, 0, smaller);
-      
-      if(oai_coll_size > resume_after) {
-        //An incomplete list is sent; append a resumptionToken element
-        Element token = createResumptionTokenElement(oai_coll_size, 0, resume_after, true);
-        //store this token
-        OAIXML.addToken(token);
-        
-        list_sets_elem.appendChild(token);
-      }
-      
-      return getMessage(list_sets_elem);
-    } 
-    
-    // The url should contain only one param called resumptionToken
-    // This is requesting a subsequent part of a list, following a previously sent incomplete list
-    param = (Element)params.item(0);
-    String param_name = param.getAttribute(OAIXML.NAME);
-    if (!param_name.equals(OAIXML.RESUMPTION_TOKEN)) {
-      //Bad argument
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    }
-    //get the token
-    String token = param.getAttribute(OAIXML.VALUE);
-    //validate the token string (the string has already been decoded in OAIServer, e.g., 
-    // replace %3A with ':')
-    if(OAIXML.containsToken(token) == false) {
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_RESUMPTION_TOKEN, ""));
-    }
-    //take out the cursor value, which is the size of previously sent list
-    int index = token.indexOf(":");
-    int cursor = Integer.parseInt(token.substring(index + 1));
-    Element token_elem = null;
-    
-    // are we sending the final part of a complete list?
-    if(cursor + resume_after >= oai_coll_size) {
-      //Yes, we are.
-      //append required sets to list_sets_elem (list is complete)
-      getSets(list_sets_elem, collection_list, cursor, oai_coll_size);
-      //An incomplete list is sent; append a resumptionToken element
-      token_elem = createResumptionTokenElement(oai_coll_size, cursor, -1, false);
-      list_sets_elem.appendChild(token_elem); 
-    } else {
-      //No, we are not.
-      //append required sets to list_sets_elem (list is incomplete)
-      getSets(list_sets_elem, collection_list, cursor, cursor + resume_after);
-      token_elem = createResumptionTokenElement(oai_coll_size, cursor, cursor + resume_after, true);
-      //store this token
-      OAIXML.addToken(token_elem);
-      list_sets_elem.appendChild(token_elem);                        
-    }
-    return getMessage(list_sets_elem);
-  }  
-    private Element createResumptionTokenElement(int total_size, int cursor, int so_far_sent, boolean set_expiration, String metadata_prefix) {
-    Element token = OAIXML.createElement(OAIXML.RESUMPTION_TOKEN);
-    token.setAttribute(OAIXML.COMPLETE_LIST_SIZE, "" + total_size);
-    token.setAttribute(OAIXML.CURSOR, "" + cursor);
-    
-    if(set_expiration) {
-      /** read the resumptionTokenExpiration element in OAIConfig.xml and get the specified time value
-       *  Use the time value plus the current system time to get the expiration date string.
-       */
-	String expiration_date = OAIXML.getTime(System.currentTimeMillis() + OAIXML.getTokenExpiration()); // in milliseconds
-      token.setAttribute(OAIXML.EXPIRATION_DATE, expiration_date);
-    }
-    
-    if(so_far_sent > 0) {
-      //the format of resumptionToken is not defined by the OAI-PMH and should be
-      //considered opaque by the harvester (in other words, strictly follow what the
-      //data provider has to offer
-      //Here, we make use of the uniqueness of the system time
-	String tokenValue = OAIXML.GS3OAI + System.currentTimeMillis() + ":" + so_far_sent;	
-	if(!metadata_prefix.equals("")) {
-	    tokenValue = tokenValue + ":" + metadata_prefix;
-	}
-	GSXML.setNodeText(token, tokenValue);
-    }
-    return token;
-  }
-    
-    private Element createResumptionTokenElement(int total_size, int cursor, int so_far_sent, boolean set_expiration) {
-	return createResumptionTokenElement(total_size, cursor, so_far_sent, set_expiration, ""); // empty metadata_prefix
-    }
+
 
   /** if the param_map contains strings other than those in valid_strs, return false;
    *  otherwise true.
    */
-  private boolean isValidParam(HashMap<String, String> param_map, HashSet<String> valid_strs) {
+  private boolean areAllParamsValid(HashMap<String, String> param_map, HashSet<String> valid_strs) {
     ArrayList<String> param_list = new ArrayList<String>(param_map.keySet());
     for(int i=0; i<param_list.size(); i++) {
+      logger.error("param, key =  "+param_list.get(i)+", value = "+param_map.get(param_list.get(i)));
       if (valid_strs.contains(param_list.get(i)) == false) {
         return false;
@@ -351,6 +355,7 @@
     return true;
   }
-  private Element doListIdentifiers(Element msg) {
-    // option: from, until, set, metadataPrefix, resumptionToken
+
+  private Element doListIdentifiersOrRecords(Element req, String verb, String record_type) {
+    // options: from, until, set, metadataPrefix, resumptionToken
     // exceptions: badArgument, badResumptionToken, cannotDisseminateFormat, noRecordMatch, and noSetHierarchy
     HashSet<String> valid_strs = new HashSet<String>();
@@ -360,451 +365,307 @@
     valid_strs.add(OAIXML.METADATA_PREFIX);
     valid_strs.add(OAIXML.RESUMPTION_TOKEN);
-        
-    Element list_identifiers = OAIXML.createElement(OAIXML.LIST_IDENTIFIERS);
-    Element req = (Element)GSXML.getChildByTagName(msg, GSXML.REQUEST_ELEM);
-    if (req == null) {       logger.error("req is null");       return null;     }
-    NodeList params = GSXML.getChildrenByTagName(req, OAIXML.PARAM);
-    String coll_name = "";
-    String token = "";
-    
-    HashMap<String, String> param_map = OAIXML.getParamMap(params);    
-    if (!isValidParam(param_map, valid_strs)) {
-	logger.error("One of the params is invalid");
-	return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    } 
-    // param keys are valid, but if there are any date params, check they're of the right format
-    String from = param_map.get(OAIXML.FROM);
-    if(from != null) {	
+    
+    Document result_doc = this.converter.newDOM();
+    Element result_element = result_doc.createElement(verb);
+    boolean result_token_needed = false; // does this result need to include a
+    // resumption token
+
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+
+    HashMap<String, String> param_map = GSXML.getParamMap(params); 
+    
+    // are all the params valid?
+    if (!areAllParamsValid(param_map, valid_strs)) {
+      logger.error("One of the params is invalid");
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "There was an invalid parameter");
+      // TODO, need to tell the user which one was invalid ??
+    }     
+    
+    // Do we have a resumption token??
+    String token = null;
+    String from = null;
+    String until = null;
+    boolean set_requested = false;
+    String set_spec_str = null;
+    String prefix_value = null;
+    int cursor = 0;
+    int current_cursor = 0;
+    String current_set = null;
+    
+    int total_size = -1; // we are only going to set this in resumption 
+    // token if it is easy to work out, i.e. not sending extra requests to
+    // MR just to calculate total size
+
+    if(param_map.containsKey(OAIXML.RESUMPTION_TOKEN)) {
+      // Is it an error to have other arguments? Do we need to check to make sure that resumptionToken is the only arg??
+      // validate resumptionToken
+      token = param_map.get(OAIXML.RESUMPTION_TOKEN);
+      logger.info("has resumptionToken " + token);
+      if(OAIResumptionToken.isValidToken(token) == false) {
+	logger.error("token is not valid");
+        return OAIXML.createErrorMessage(OAIXML.BAD_RESUMPTION_TOKEN, "");
+      }
+      result_token_needed = true; // we always need to send a token back if we have started with one. It may be empty if we are returning the end of the list
+      // initialise the request params from the stored token data
+      HashMap<String, String> token_data = OAIResumptionToken.getTokenData(token);
+      from = token_data.get(OAIXML.FROM);
+      until = token_data.get(OAIXML.UNTIL);
+      set_spec_str = token_data.get(OAIXML.SET);
+      if (set_spec_str != null) {
+	set_requested = true;
+      }
+      prefix_value = token_data.get(OAIXML.METADATA_PREFIX);
+      current_set = token_data.get(OAIResumptionToken.CURRENT_SET);
+      try {
+	cursor = Integer.parseInt(token_data.get(OAIXML.CURSOR));
+	cursor = cursor + resume_after; // increment cursor
+	current_cursor = Integer.parseInt(token_data.get(OAIResumptionToken.CURRENT_CURSOR));
+      } catch (NumberFormatException e) {
+	logger.error("tried to parse int from cursor data and failed");
+      }
+      
+    }
+    else {
+      // no resumption token, lets check the other params
+      // there must be a metadataPrefix
+      if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
+	logger.error("metadataPrefix param required");
+	return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "metadataPrefix param required");
+      }
+
+      //if there are any date params, check they're of the right format
+      from = param_map.get(OAIXML.FROM);
+      if(from != null) {	
 	Date from_date = OAIXML.getDate(from);
 	if(from_date == null) {
-	    logger.error("invalid date: " + from);
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+	  logger.error("invalid date: " + from);
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid format for "+ OAIXML.FROM);
 	}
-    }
-    String until = param_map.get(OAIXML.UNTIL);
-    if(until != null) {	
+      }
+      until = param_map.get(OAIXML.UNTIL);
+      if(until != null) {	
 	Date until_date = OAIXML.getDate(until);
 	if(until_date == null) {
-	    logger.error("invalid date: " + until);
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+	  logger.error("invalid date: " + until);
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid format for "+ OAIXML.UNTIL);
 	} 
-    }    
-    if(from != null && until != null) { // check they are of the same date-time format (granularity)
+      }    
+      if(from != null && until != null) { // check they are of the same date-time format (granularity)
 	if(from.length() != until.length()) {
-	    logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+	  logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "The request has different granularities (date-time formats) for the From and Until date parameters.");
 	}
-    }
-
-    //ask the message router for a list of oai collections
-    //NodeList oai_coll = collection_list; //getOAICollectionList();
-    int oai_coll_size = collection_list.getLength();
-    if (oai_coll_size == 0) {
-      logger.info("returned oai collection list is empty");
-      return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }
-    
-    //Now we check if the optional argument 'set' has been specified in the params; if so,
-    //whether the specified setSpec is supported by this repository 
-    boolean request_set = param_map.containsKey(OAIXML.SET);
-    if(request_set == true) {
-      boolean set_supported = false;
-      String set_spec_str = param_map.get(OAIXML.SET);
-      // get the collection name
-      //if setSpec is supported by this repository, it must be in the form: site_name:coll_name
-      String[] strs = splitSetSpec(set_spec_str);
-      coll_name = strs[1];
-
-      for(int i=0; i<oai_coll_size; i++) {
-        if(set_spec_str.equals(((Element)collection_list.item(i)).getAttribute(OAIXML.NAME))) {
-          set_supported = true;
-        }
-      }
-      if(set_supported == false) {
-        logger.error("requested set is not found in this repository");
-        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-      }
-    }
-    
-    //Is there a resumptionToken included which is requesting an incomplete list?
-    if(param_map.containsKey(OAIXML.RESUMPTION_TOKEN)) {
-        // validate resumptionToken
-      token = param_map.get(OAIXML.RESUMPTION_TOKEN);
-      logger.info("has resumptionToken" + token);
-      if(OAIXML.containsToken(token) == false) {
-        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_RESUMPTION_TOKEN, ""));
-      }
-    }
-    
-    // Custom test that expects a metadataPrefix comes here at end so that the official params can
-    // be tested first for errors and their error responses sent off. Required for OAI validation
-    if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
-      logger.error("contains invalid params or no metadataPrefix");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    }    
-    
-    //Now that we got a prefix, check and see if it's supported by this repository
-    String prefix_value = param_map.get(OAIXML.METADATA_PREFIX);
-    if (containsMetadataPrefix(prefix_value) == false) {
-      logger.error("requested prefix is not found in OAIConfig.xml");
-      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
-    }
-
-    //Now that all validation has been done, I hope, we can send request to the message router
-    Element result = null;
-    String verb = req.getAttribute(OAIXML.TO); 
-    NodeList param_list = req.getElementsByTagName(OAIXML.PARAM);
-    ArrayList<Element> retain_param_list = new ArrayList<Element>();
-    for (int j=0; j<param_list.getLength(); j++) {
-      Element e = OAIXML.duplicateElement(msg.getOwnerDocument(), (Element)param_list.item(j), true);
-      retain_param_list.add(e);
-    }
-
-    //re-organize the request element
-    // reset the 'to' attribute
-    if (request_set == false) {
-      logger.info("requesting identifiers of all collections");
-      for(int i=0; i<oai_coll_size; i++) {
-        if(req == null) {
-          req = msg.getOwnerDocument().createElement(GSXML.REQUEST_ELEM);
-          msg.appendChild(req);
-          for (int j=0; j<retain_param_list.size(); j++) {
-            req.appendChild(retain_param_list.get(j));
-          }
-        }
-        String full_name = ((Element)collection_list.item(i)).getAttribute(OAIXML.NAME);
-        coll_name = full_name.substring(full_name.indexOf(":") + 1);
-        req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
-        Node n = mr.process(msg);
-	Element e = converter.nodeToElement(n);
-        result = collectAll(result, e, verb, OAIXML.HEADER);
-        
-        //clear the content of the old request element
-        msg.removeChild(req);
-        req = null;
-      }      
-    } else {
-      req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
-      Node result_node = mr.process(msg);
-      result = converter.nodeToElement(result_node);
-    }
-    
-    if (result == null) {
-      logger.info("message router returns null");
-      return getMessage(OAIXML.createErrorElement("Internal service returns null", ""));
-    }
-    Element res = (Element)GSXML.getChildByTagName(result, OAIXML.RESPONSE);
+      }
+  
+      // check the set arg is a set we know about
+      set_requested = param_map.containsKey(OAIXML.SET);
+      set_spec_str = null;
+      if(set_requested == true) {
+	set_spec_str = param_map.get(OAIXML.SET);
+	if (!this.set_set.contains(set_spec_str)) {
+	  // the set is not one we know about
+	  logger.error("requested set is not found in this repository");
+	  return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "invalid set parameter");	
+	  
+	}
+      }
+      // Is the metadataPrefix arg one this repository supports?
+      prefix_value = param_map.get(OAIXML.METADATA_PREFIX);
+      if (repositorySupportsMetadataPrefix(prefix_value) == false) {
+	logger.error("requested metadataPrefix is not found in OAIConfig.xml");
+	return OAIXML.createErrorMessage(OAIXML.CANNOT_DISSEMINATE_FORMAT, "metadata format "+prefix_value+" not supported by this repository");
+      }
+      
+    } // else no resumption token, check other params    
+    
+    // Whew. Now we have validated the params, we can work on doing the actual
+    // request
+
+
+    Document doc = this.converter.newDOM();
+    Element mr_msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element mr_req = doc.createElement(GSXML.REQUEST_ELEM);
+    // TODO does this need a type???
+    mr_msg.appendChild(mr_req);
+
+    // copy in the from/until params if there
+    if (from != null) {
+      mr_req.appendChild(GSXML.createParameter(doc, OAIXML.FROM, from));
+    }
+    if (until != null) {
+      mr_req.appendChild(GSXML.createParameter(doc, OAIXML.UNTIL, until));
+    }
+    // add metadataPrefix
+    mr_req.appendChild(GSXML.createParameter(doc, OAIXML.METADATA_PREFIX, prefix_value));
+	
+    // do we have a set???
+    // if no set, we send to all collections in the collection list
+    // if super set, we send to all collections in super set list
+    // if a single collection, send to it
+    // if a subset, send to the collection
+    Vector<String> current_coll_list = null;
+    boolean single_collection = false;
+    if (set_requested == false) {
+      // just do all colls
+      current_coll_list = collection_name_list;
+    }
+    else if (has_super_colls && super_coll_map.containsKey(set_spec_str)) {
+      current_coll_list = super_coll_map.get(set_spec_str);
+    }
+    else {
+      current_coll_list = new Vector<String>();
+      if (set_spec_str.indexOf(":") != -1) {
+	// we have a subset
+	//add the set param back into the request, but send the request to the collection
+	String col_name = set_spec_str.substring(0, set_spec_str.indexOf(":"));
+	current_coll_list.add(col_name);
+	mr_req.appendChild(GSXML.createParameter(doc, OAIXML.SET, set_spec_str));
+	single_collection = true;
+      }
+      else {
+	// it must be a single collection name
+	current_coll_list.add(set_spec_str);
+	single_collection = true;
+      }
+    }
+
+    int num_collected_records = 0;
+    int start_point = current_cursor; // may not be 0 if we are using a resumption token
+    String resumption_collection = "";
+    boolean empty_result_token = false; // if we are sending the last part of a list, then the token value will be empty
+    
+    // iterate through the list of collections and send the request to each
+
+    int start_coll=0;
+    if (current_set != null) {
+      // we are resuming a previous request, need to locate the first collection
+      for (int i=0; i<current_coll_list.size(); i++) {
+	if (current_set.equals(current_coll_list.get(i))) {
+	  start_coll = i;
+	  break;
+	}
+      }
+    }
+    
+    for (int i=start_coll; i<current_coll_list.size(); i++) {
+      String current_coll = current_coll_list.get(i);
+      mr_req.setAttribute(GSXML.TO_ATT, current_coll+"/"+verb);
+
+      Element result = (Element)mr.process(mr_msg);
+      logger.error(verb+ " result for coll "+current_coll);
+      logger.error(this.converter.getPrettyString(result));
+      if (result == null) {
+	logger.info("message router returns null");
+	// do what??? carry on? fail??
+	return OAIXML.createErrorMessage("Internal service returns null", "");
+      }
+      Element res = (Element)GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
       if(res == null) {
         logger.info("response element in xml_result is null");
-        return getMessage(OAIXML.createErrorElement("Internal service returns null", ""));
-      }
-    NodeList header_list = res.getElementsByTagName(OAIXML.HEADER);
-    int num_headers = header_list.getLength();
-    if(num_headers == 0) {
-      logger.info("message router returns 0 headers.");
-      return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }      
-
-    //The request coming in does not contain a token, but we have to check the resume_after value and see if we need to issue a resumption token and
-    //      save the token as well.
-    if (token.equals("") == true) {
-      if(resume_after < 0 || num_headers <= resume_after) {
-        //send the whole list of records
-        return result;
-      }
+        return OAIXML.createErrorMessage("Internal service returns null", "");
+      }
+      NodeList record_list = res.getElementsByTagName(record_type);
+      int num_records = record_list.getLength();
+      if(num_records == 0) {
+	logger.info("message router returns 0 records for coll "+current_coll);
+	continue; // try the next collection
+      }  
+      if (single_collection) {
+	total_size = num_records;
+      }
+      int records_to_add = (resume_after > 0 ? resume_after - num_collected_records : num_records);
+      if (records_to_add > (num_records-start_point)) {
+	records_to_add = num_records-start_point;
+      }
+      addRecordsToList(result_doc, result_element, record_list, start_point, records_to_add);
+      num_collected_records += records_to_add;
       
-      //append required number of records (may be a complete or incomplete list)
-      getRecords(list_identifiers, header_list, 0, resume_after);
-      //An incomplete list is sent; append a resumptionToken element
-      Element token_elem = createResumptionTokenElement(num_headers, 0, resume_after, true);
-      //store this token
-      OAIXML.addToken(token_elem);
-       
-      list_identifiers.appendChild(token_elem);
-      return getMessage(list_identifiers);
-    } 
-        
-    if (token.equals("") == false) {
-      //get an appropriate number of records (partial list) according to the token
-      //take out the cursor value, which is the size of previously sent list
-      int index = token.indexOf(":");
-      int cursor = Integer.parseInt(token.substring(index + 1));
-      Element token_elem = null;
+      // do we need to stop here, and do we need to issue a resumption token?
+      if (resume_after > 0 && num_collected_records == resume_after) {
+	// we have finished collecting records at the moment.
+	// but are we conincidentally at the end? or are there more to go?
+	if (records_to_add < (num_records - start_point)) {
+	  // we have added less than this collection had
+	  start_point += records_to_add;
+	  resumption_collection = current_coll;
+	  result_token_needed = true;
+	}
+	else {
+	  // we added all this collection had to offer
+	  // is there another collection in the list??
+	  if (i<current_coll_list.size()-1) {
+	    result_token_needed = true;
+	    start_point = 0;
+	    resumption_collection = current_coll_list.get(i+1);
+	  } 
+	  else {
+	    // we have finished one collection and there are no more collection
+	    // if we need to send a resumption token (in this case, only because we started with one, then it will be empty
+	    logger.error("at end of list, need empty result token");
+	    empty_result_token = true;
+	  }
+	}
+	break;
+      }
+      start_point = 0; // only the first one will have start non-zero, if we
+      // have a resumption token
       
-      // are we sending the final part of a complete list?
-      if(cursor + resume_after >= num_headers) {
-        //Yes, we are.
-        //append required records to list_records (list is complete)
-        getRecords(list_identifiers, header_list, cursor, num_headers);
-        //An incomplete list is sent; append a resumptionToken element
-        token_elem = createResumptionTokenElement(num_headers, cursor, -1, false);
-        list_identifiers.appendChild(token_elem);
+    } // for each collection
+
+    if (num_collected_records ==0) {
+      // there were no matching results
+      return OAIXML.createErrorMessage(OAIXML.NO_RECORDS_MATCH, "");
+    }
+
+    if (num_collected_records < resume_after) {
+      // we have been through all collections, and there are no more
+      // if we need a result token - only because we started with one, so we need to send an empty one, then make sure everyone knows we are just sending an empty one
+      if (result_token_needed) {
+	empty_result_token = true;
+      }
+    }
+    
+    if (result_token_needed) {
+      // we need a resumption token
+      if (empty_result_token) {
+	logger.error("have empty result token");
+	token = "";
       } else {
-        //No, we are not.
-        //append required records to list_records (list is incomplete)
-        getRecords(list_identifiers, header_list, cursor, cursor + resume_after);
-        token_elem = createResumptionTokenElement(num_headers, cursor, cursor + resume_after, true);
-        //store this token
-        OAIXML.addToken(token_elem);
-        list_identifiers.appendChild(token_elem);
-      }      
-      
-      return getMessage(list_identifiers);
-    }//end of if(!token.equals("")) 
-
-    return result;
-  }
-  private Element doListRecords(Element msg){
-    logger.info("");
-    // option: from, until, set, metadataPrefix, and resumptionToken
-    // exceptions: badArgument, badResumptionToken, cannotDisseminateFormat, noRecordMatch, and noSetHierarchy
-    HashSet<String> valid_strs = new HashSet<String>();
-    valid_strs.add(OAIXML.FROM);
-    valid_strs.add(OAIXML.UNTIL);
-    valid_strs.add(OAIXML.SET);
-    valid_strs.add(OAIXML.METADATA_PREFIX);
-    valid_strs.add(OAIXML.RESUMPTION_TOKEN);
-
-    Element list_records = OAIXML.createElement(OAIXML.LIST_RECORDS);
-    Element req = (Element)GSXML.getChildByTagName(msg, GSXML.REQUEST_ELEM);
-    if (req == null) {       logger.error("req is null");       return null;     }
-    NodeList params = GSXML.getChildrenByTagName(req, OAIXML.PARAM);
-
-    String coll_name = "";
-    String token = "";
-    
-    if(params.getLength() == 0) {
-      logger.error("must at least have the metadataPrefix parameter, can't be none");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    }
-    
-    HashMap<String, String> param_map = OAIXML.getParamMap(params);    
-    if (!isValidParam(param_map, valid_strs)) {
-	logger.error("One of the params is invalid");
-	return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-    }
-    // param keys are valid, but if there are any date params, check they're of the right format
-    String from = param_map.get(OAIXML.FROM);
-    if(from != null) {	
-	Date from_date = OAIXML.getDate(from);
-	if(from_date == null) {
-	    logger.error("invalid date: " + from);
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+	if (token != null) {
+	  // we had a token for this request, we can just update it
+	  token = OAIResumptionToken.updateToken(token, ""+cursor, resumption_collection, ""+start_point);
+	} else {
+	  // we are generating a new one
+	  token = OAIResumptionToken.createAndStoreResumptionToken(set_spec_str, prefix_value, from, until, ""+cursor, resumption_collection, ""+start_point );
 	}
-    }
-    String until = param_map.get(OAIXML.UNTIL);
-    Date until_date = null;
-    if(until != null) {	
-	until_date = OAIXML.getDate(until);
-	if(until_date == null) {
-	    logger.error("invalid date: " + until);
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-	} 
-    }
-    if(from != null && until != null) { // check they are of the same date-time format (granularity)
-	if(from.length() != until.length()) {
-	    logger.error("The request has different granularities (date-time formats) for the From and Until date parameters.");
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-	}
-    }
-    
-    //ask the message router for a list of oai collections
-    //NodeList oai_coll = getOAICollectionList();
-    int oai_coll_size = collection_list.getLength();
-    if (oai_coll_size == 0) {
-      logger.info("returned oai collection list is empty");
-      return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }
-    
-    //Now we check if the optional argument 'set' has been specified in the params; if so,
-    //whether the specified setSpec is supported by this repository 
-    boolean request_set = param_map.containsKey(OAIXML.SET);
-    if(request_set == true) {
-      boolean set_supported = false;
-      String set_spec_str = param_map.get(OAIXML.SET);
-      // get the collection name
-      //if setSpec is supported by this repository, it must be in the form: site_name:coll_name
-      String[] strs = splitSetSpec(set_spec_str);
-//    name_of_site = strs[0];
-      coll_name = strs[1];
-      //logger.info("param contains set: "+coll_name);
-
-      for(int i=0; i<oai_coll_size; i++) {
-        if(set_spec_str.equals(((Element)collection_list.item(i)).getAttribute(OAIXML.NAME))) {
-          set_supported = true;
-        }
-      }
-      if(set_supported == false) {
-        logger.error("requested set is not found in this repository");
-        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-      }
-    }
-    
-    //Is there a resumptionToken included which is requesting an incomplete list?
-    if(param_map.containsKey(OAIXML.RESUMPTION_TOKEN)) {
-        // validate resumptionToken
-        //if (the token value is not found in the token xml file) {
-        //  return getMessage(OAIXML.createErrorElement(OAIXML.BAD_RESUMPTION_TOKEN, ""));
-        //} else {
-        //   use the request to get a complete list of records from the message router
-      //    and issue the subsequent part of that complete list according to the token.
-      //    store a new token if necessary.
-        //}      
-      token = param_map.get(OAIXML.RESUMPTION_TOKEN);
-      logger.info("has resumptionToken: " + token);
-      if(OAIXML.containsToken(token) == false) {
-        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_RESUMPTION_TOKEN, ""));
-      }
-    }
-
-    // Moved the additional custom test that mandates the metadataPrefix here, since official
-    // errors should be caught first, so that their error responses can be sent off first
-    // such that GS2's oaiserver will validate properly.
-    if (!param_map.containsKey(OAIXML.METADATA_PREFIX)) {
-	if(!token.equals("")) { // resumptiontoken
-	    int lastIndex = token.lastIndexOf(":");
-	    if(lastIndex != token.indexOf(":")) { // if a meta_prefix is suffixed to the usual token,
-		// put that in the map and remove it from the end of the stored token
-		String meta_prefix = token.substring(lastIndex+1);
-		param_map.put(OAIXML.METADATA_PREFIX, meta_prefix);
-		token = token.substring(0, lastIndex);
-		param_map.put(OAIXML.RESUMPTION_TOKEN, token);
-
-		// Add to request <param name="metadataPrefix" value="oai_dc"/>
-		// need to add metaprefix as param to request, else a request 
-		// for subsequent records when working with resumption tokens will fail
-		Element paramEl = req.getOwnerDocument().createElement(OAIXML.PARAM);
-		paramEl.setAttribute(OAIXML.NAME, OAIXML.METADATA_PREFIX);
-		paramEl.setAttribute(OAIXML.VALUE, meta_prefix);
-		req.appendChild(paramEl);
-	    }
-	} else { // no metadata_prefix
-
-	    // it must have a metadataPrefix
-	    /** Here I disagree with the OAI specification: even if a resumptionToken is 
-	     *  included in the request, the metadataPrefix is a must argument. Otherwise
-	     *  how would we know what metadataPrefix the harvester requested in his last request?
-	     */
-	    logger.error("no metadataPrefix");
-	    return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-	}
-    }
-    
-    //Now that we got a prefix, check and see if it's supported by this repository
-    String prefix_value = param_map.get(OAIXML.METADATA_PREFIX);
-    if (containsMetadataPrefix(prefix_value) == false) {
-	logger.error("requested prefix is not found in OAIConfig.xml");
-	return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
-    }
-    
-
-    //Now that all validation has been done, I hope, we can send request to the message router
-    Element result = null;
-    String verb = req.getAttribute(OAIXML.TO); 
-    NodeList param_list = req.getElementsByTagName(OAIXML.PARAM);
-    ArrayList<Element> retain_param_list = new ArrayList<Element>();
-    for (int j=0; j<param_list.getLength(); j++) {
-      Element e = OAIXML.duplicateElement(msg.getOwnerDocument(), (Element)param_list.item(j), true);
-      retain_param_list.add(e);
-    }
-
-    //re-organize the request element
-    // reset the 'to' attribute
-    if (request_set == false) {
-      //coll_name could be "", which means it's requesting all records of all collections
-      //we send a request to each collection asking for its records
-      for(int i=0; i<oai_coll_size; i++) {
-        if(req == null) {
-          req = msg.getOwnerDocument().createElement(GSXML.REQUEST_ELEM);
-          msg.appendChild(req);
-          for (int j=0; j<retain_param_list.size(); j++) {
-            req.appendChild(retain_param_list.get(j));
-          }
-        }
-        String full_name = ((Element)collection_list.item(i)).getAttribute(OAIXML.NAME);
-        coll_name = full_name.substring(full_name.indexOf(":") + 1);
-        req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
-        //logger.info(GSXML.xmlNodeToString(req));
-        Node n = mr.process(msg);
-	Element e = converter.nodeToElement(n);
-        result = collectAll(result, e, verb, OAIXML.RECORD);
-
-        //clear the content of the old request element
-        msg.removeChild(req);
-        req = null;
-      }      
-    } else {
-      req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
-
-      Node result_node = mr.process(msg);
-      result = converter.nodeToElement(result_node);
-    }
-    
-    if (result == null) {
-      logger.info("message router returns null");
-      return getMessage(OAIXML.createErrorElement("Internal service returns null", ""));
-    }
-    Element res = (Element)GSXML.getChildByTagName(result, OAIXML.RESPONSE);
-      if(res == null) {
-        logger.info("response element in xml_result is null");
-        return getMessage(OAIXML.createErrorElement("Internal service returns null", ""));
-      }
-    NodeList record_list = res.getElementsByTagName(OAIXML.RECORD);
-    int num_records = record_list.getLength();
-    if(num_records == 0) {
-      logger.info("message router returns 0 records.");
-      return getMessage(OAIXML.createErrorElement(OAIXML.NO_RECORDS_MATCH, ""));
-    }      
-
-    //The request coming in does not contain a token, but we have to check the resume_after value and see if we need to issue a resumption token and
-    //      save the token as well.
-    if (token.equals("") == true) {
-      if(resume_after < 0 || num_records <= resume_after) {
-        //send the whole list of records
-        return result;
-      }
-      
-      //append required number of records (may be a complete or incomplete list)
-      getRecords(list_records, record_list, 0, resume_after);
-      //An incomplete list is sent; append a resumptionToken element
-      Element token_elem = createResumptionTokenElement(num_records, 0, resume_after, true, param_map.get(OAIXML.METADATA_PREFIX));
-      //store this token
-      OAIXML.addToken(token_elem);
-       
-      list_records.appendChild(token_elem);
-      return getMessage(list_records);
-    } 
-        
-    if (token.equals("") == false) {
-      //get an appropriate number of records (partial list) according to the token
-      //take out the cursor value, which is the size of previously sent list
-      int index = token.indexOf(":");
-      int cursor = Integer.parseInt(token.substring(index + 1));
-      Element token_elem = null;
-      
-      // are we sending the final part of a complete list?
-      if(cursor + resume_after >= num_records) {
-        //Yes, we are.
-        //append required records to list_records (list is complete)
-        getRecords(list_records, record_list, cursor, num_records);
-        //An incomplete list is sent; append a resumptionToken element
-	token_elem = createResumptionTokenElement(num_records, cursor, -1, false, param_map.get(OAIXML.METADATA_PREFIX));
-	list_records.appendChild(token_elem);
-
+      }
+
+      // result token XML
+      long expiration_date = -1;
+      if (empty_result_token) {
+	// we know how many records in total as we have sent them all
+	total_size = cursor+num_collected_records;
       } else {
-        //No, we are not.
-        //append required records to list_records (list is incomplete)
-        getRecords(list_records, record_list, cursor, cursor + resume_after);
-        token_elem = createResumptionTokenElement(num_records, cursor, cursor + resume_after, true, param_map.get(OAIXML.METADATA_PREFIX));
-        //store this token
-        OAIXML.addToken(token_elem);
-        list_records.appendChild(token_elem);
-      }      
-      
-      return getMessage(list_records);
-    }//end of if(!token.equals("")) 
-
-    return result;//a backup return
-  } 
+	// non-empty token, set the expiration date
+	expiration_date = OAIResumptionToken.getExpirationDate(token);
+      }
+      Element token_elem = OAIXML.createResumptionTokenElement(result_doc, token, total_size, cursor, expiration_date); 
+      // OAIXML.addToken(token_elem); // store it
+      result_element.appendChild(token_elem); // add to the result
+    }
+  
+
+    return getMessage(result_doc, result_element);
+  }
+
+  private void addRecordsToList(Document doc, Element result_element, NodeList
+				record_list, int start_point, int num_records) {
+    int end_point = start_point + num_records;
+    for (int i=start_point; i<end_point; i++) {
+      result_element.appendChild(doc.importNode(record_list.item(i), true));
+    }
+  }
+
+
   // method exclusively used by doListRecords/doListIdentifiers
   private void getRecords(Element verb_elem, NodeList list, int start_point, int end_point) {
@@ -818,7 +679,7 @@
       return msg;
     }
-    Element res_in_result = (Element)GSXML.getChildByTagName(result, OAIXML.RESPONSE);
+    Element res_in_result = (Element)GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM);
     if(res_in_result == null) { // return the results of all other collections accumulated so far
-	return msg;
+      return msg;
     }
     Element verb_elem = (Element)GSXML.getChildByTagName(res_in_result, verb);
@@ -836,37 +697,44 @@
     return result;
   }
-  /** there are three possible exception conditions: bad argument, idDoesNotExist, and noMetadataFormats.
-    * The first one is handled here, and the last two are processed by OAIPMH.
-    */
-  private Element doListMetadataFormats(Element msg) {
+
+  
+  /** there are three possible exception conditions: bad argument, idDoesNotExist, and noMetadataFormat.
+   * The first one is handled here, and the last two are processed by OAIPMH.
+   */
+  private Element doListMetadataFormats(Element req) {
     //if the verb is ListMetadataFormats, there could be only one parameter: identifier
     //, or there is no parameter; otherwise it is an error
     //logger.info("" + this.converter.getString(msg));
     
-    Element list_metadata_formats = OAIXML.createElement(OAIXML.LIST_METADATA_FORMATS);
-
-    Element req = (Element)GSXML.getChildByTagName(msg, GSXML.REQUEST_ELEM);
-    if (req == null) {       logger.error("");       return null;     }
-    NodeList params = GSXML.getChildrenByTagName(req, OAIXML.PARAM);
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
     Element param = null;
+    Document lmf_doc = this.converter.newDOM();
     if(params.getLength() == 0) {
       //this is requesting metadata formats for the whole repository
       //read the oaiConfig.xml file, return the metadata formats specified there.
+      if (this.listmetadataformats_response != null) {
+	// we have already created it
+	return this.listmetadataformats_response;
+      }
+
+      Element list_metadata_formats = lmf_doc.createElement(OAIXML.LIST_METADATA_FORMATS);
+      
       Element format_list = (Element)GSXML.getChildByTagName(oai_config, OAIXML.LIST_METADATA_FORMATS);
       if(format_list == null) {
 	logger.error("OAIConfig.xml must contain the supported metadata formats");
-	return getMessage(list_metadata_formats);
+	// TODO this is internal error, what to do???
+	return getMessage(lmf_doc, list_metadata_formats);
       }
       NodeList formats = format_list.getElementsByTagName(OAIXML.METADATA_FORMAT);
       for(int i=0; i<formats.getLength(); i++) {
-	Element meta_fmt = OAIXML.createElement(OAIXML.METADATA_FORMAT);
+	Element meta_fmt = lmf_doc.createElement(OAIXML.METADATA_FORMAT);
 	Element first_meta_format = (Element)formats.item(i);
 	//the element also contains mappings, but we don't want them
-	meta_fmt.appendChild(meta_fmt.getOwnerDocument().importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.METADATA_PREFIX), true));
-	meta_fmt.appendChild(meta_fmt.getOwnerDocument().importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.SCHEMA), true));
-	meta_fmt.appendChild(meta_fmt.getOwnerDocument().importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.METADATA_NAMESPACE), true));
+	meta_fmt.appendChild(lmf_doc.importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.METADATA_PREFIX), true));
+	meta_fmt.appendChild(lmf_doc.importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.SCHEMA), true));
+	meta_fmt.appendChild(lmf_doc.importNode(GSXML.getChildByTagName(first_meta_format, OAIXML.METADATA_NAMESPACE), true));
 	list_metadata_formats.appendChild(meta_fmt);
-        }
-      return getMessage(list_metadata_formats);
+      }
+      return getMessage(lmf_doc, list_metadata_formats);
       
       
@@ -875,64 +743,61 @@
     if (params.getLength() > 1) {
       //Bad argument. Can't be more than one parameters for ListMetadataFormats verb
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
     } 
     
     // This is a request for the metadata of a particular item with an identifier
-      /**the request xml is in the form: <request>
-       *                                   <param name=.../>
-       *                                 </request>
-       *And there is a param element and one element only. (No paramList element in between).
-       */
-      param = (Element)params.item(0);
-      String param_name = param.getAttribute(OAIXML.NAME);
-      String identifier = "";
-      if (!param_name.equals(OAIXML.IDENTIFIER)) {
-        //Bad argument
-        return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
-      } else {
-        identifier = param.getAttribute(OAIXML.VALUE);
-        // the identifier is in the form: <site_name>:<coll_name>:<OID>
-        // so it must contain at least two ':' characters
-        String[] strs = identifier.split(":");
-        if(strs == null || strs.length < 3) {
-          // the OID may also contain ':'
-          logger.error("identifier is not in the form site:coll:id" + identifier);
-          return getMessage(OAIXML.createErrorElement(OAIXML.ID_DOES_NOT_EXIST, ""));
-        }
+    /**the request xml is in the form: <request>
+     *                                   <param name=.../>
+     *                                 </request>
+     *And there is a param element and one element only. (No paramList element in between).
+     */
+    param = (Element)params.item(0);
+    String param_name = param.getAttribute(GSXML.NAME_ATT);
+    String identifier = "";
+    if (!param_name.equals(OAIXML.IDENTIFIER)) {
+      //Bad argument
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
+    } 
+      
+    identifier = param.getAttribute(GSXML.VALUE_ATT);
+    // the identifier is in the form: <coll_name>:<OID>
+    // so it must contain at least two ':' characters
+    String[] strs = identifier.split(":");
+    if(strs == null || strs.length < 2) {
+      // the OID may also contain ':'
+      logger.error("identifier is not in the form coll:id" + identifier);
+      return OAIXML.createErrorMessage(OAIXML.ID_DOES_NOT_EXIST, "");
+    }
         
-        // send request to message router
-        // get the names
-        strs = splitNames(identifier);
-	if(strs == null || strs.length < 3) {
-	    logger.error("identifier is not in the form site:coll:id" + identifier);
-	    return getMessage(OAIXML.createErrorElement(OAIXML.ID_DOES_NOT_EXIST, ""));
-	}
-        String name_of_site = strs[0];
-        String coll_name = strs[1];
-        String oid = strs[2];
-
-        //re-organize the request element
-        // reset the 'to' attribute
-        String verb = req.getAttribute(OAIXML.TO);
-        req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
-        // reset the identifier element
-        param.setAttribute(OAIXML.NAME, OAIXML.OID);
-        param.setAttribute(OAIXML.VALUE, oid);
-
-        //Now send the request to the message router to process
-	Node result_node = mr.process(msg);
-	return converter.nodeToElement(result_node);
-      }
-  
-    
-  }
-
-
-  private void appendParam(Element req, String name, String value) {    
-        Element param = req.getOwnerDocument().createElement(OAIXML.PARAM);
-        param.setAttribute(OAIXML.NAME, name);
-        param.setAttribute(OAIXML.VALUE, value);
-        req.appendChild(param);
-  }
+    // send request to message router
+    // get the names
+    strs = splitNames(identifier);
+    if(strs == null || strs.length < 2) {
+      logger.error("identifier is not in the form coll:id" + identifier);
+      return OAIXML.createErrorMessage(OAIXML.ID_DOES_NOT_EXIST, "");
+    }
+    //String name_of_site = strs[0];
+    String coll_name = strs[0];
+    String oid = strs[1];
+
+    //re-organize the request element
+    // reset the 'to' attribute
+    String verb = req.getAttribute(GSXML.TO_ATT);
+    req.setAttribute(GSXML.TO_ATT, coll_name + "/" + verb);
+    // reset the identifier element
+    param.setAttribute(GSXML.NAME_ATT, OAIXML.OID);
+    param.setAttribute(GSXML.VALUE_ATT, oid);
+
+    // TODO is this the best way to do this???? should we create a new request???
+    Element message = req.getOwnerDocument().createElement(GSXML.MESSAGE_ELEM);
+    message.appendChild(req);
+    //Now send the request to the message router to process
+    Node result_node = mr.process(message);
+    return converter.nodeToElement(result_node);
+  }
+  
+    
+
+
   private void copyNamedElementfromConfig(Element to_elem, String element_name) {
     Element original_element = (Element)GSXML.getChildByTagName(oai_config, element_name);
@@ -952,8 +817,8 @@
     if (this.identify_response != null) {
       // we have already created it
-      return getMessage(this.identify_response);
-    }
-    
-    Element identify = OAIXML.createElement(OAIXML.IDENTIFY);
+      return this.identify_response;
+    }
+    Document doc = this.converter.newDOM();
+    Element identify = doc.createElement(OAIXML.IDENTIFY);
     //do the repository name
     copyNamedElementfromConfig(identify, OAIXML.REPOSITORY_NAME);
@@ -980,5 +845,5 @@
     long earliestDatestamp = getEarliestDateStamp(collection_list);
     String earliestDatestamp_str = OAIXML.getTime(earliestDatestamp);
-    Element earliestDatestamp_elem = OAIXML.createElement(OAIXML.EARLIEST_DATESTAMP);
+    Element earliestDatestamp_elem = doc.createElement(OAIXML.EARLIEST_DATESTAMP);
     GSXML.setNodeText(earliestDatestamp_elem, earliestDatestamp_str);
     identify.appendChild(earliestDatestamp_elem);
@@ -990,7 +855,8 @@
      
     // output the oai identifier
-    Element description = OAIXML.createElement(OAIXML.DESCRIPTION);
+    Element description = doc.createElement(OAIXML.DESCRIPTION);
     identify.appendChild(description);
-    Element oaiIdentifier = OAIXML.createOAIIdentifierXML(repository_id, "lucene-jdbm-demo", "ec159e");
+    // TODO, make this a valid id
+    Element oaiIdentifier = OAIXML.createOAIIdentifierXML(doc, repository_id, "lucene-jdbm-demo", "ec159e");
     description.appendChild(oaiIdentifier);
 
@@ -998,7 +864,7 @@
     Element info = (Element)GSXML.getChildByTagName(oai_config, OAIXML.OAI_INFO);
     if (info != null) {
-      NodeList meta = GSXML.getChildrenByTagName(info, OAIXML.INFO_METADATA);
+      NodeList meta = GSXML.getChildrenByTagName(info, OAIXML.METADATA);
       if (meta != null && meta.getLength() > 0) {
-	Element gsdl = OAIXML.createGSDLElement();
+	Element gsdl = OAIXML.createGSDLElement(doc);
 	description.appendChild(gsdl);
 	for (int m = 0; m<meta.getLength(); m++) {
@@ -1009,5 +875,5 @@
     }
     this.identify_response = identify;
-    return getMessage(identify);
+    return getMessage(doc, identify);
   }
   //split setSpec (site_name:coll_name) into an array of strings
@@ -1021,22 +887,16 @@
     return strs;
   }
-  /** split the identifier into <site + collection + OID> as an array 
-      It has already been checked that the 'identifier' contains at least two ':'
-   */
+  /** split the identifier into <collection + OID> as an array 
+      It has already been checked that the 'identifier' contains at least one ':'
+  */
   private String[] splitNames(String identifier) {
     logger.info(identifier);
-    String [] strs = new String[3];
+    String [] strs = new String[2];
     int first_colon = identifier.indexOf(":");
     if(first_colon == -1) {
-	return null;
+      return null;
     }
     strs[0] = identifier.substring(0, first_colon);
-
-    String sr = identifier.substring(first_colon + 1);
-    int second_colon = sr.indexOf(":");
-    //logger.error(first_colon + "    " + second_colon);
-    strs[1] = sr.substring(0, second_colon);
-    
-    strs[2] = sr.substring(second_colon + 1);
+    strs[1] = identifier.substring(first_colon + 1);
     return strs;
   }
@@ -1044,5 +904,5 @@
    *  by checking it in the OAIConfig.xml
    */
-  private boolean containsMetadataPrefix(String prefix_value) {
+  private boolean repositorySupportsMetadataPrefix(String prefix_value) {
     NodeList prefix_list = oai_config.getElementsByTagName(OAIXML.METADATA_PREFIX);
     
@@ -1054,12 +914,13 @@
     return false;
   }
-  private Element doGetRecord(Element msg){
+  private Element doGetRecord(Element req){
     logger.info("");
     /** arguments:
         identifier: required
         metadataPrefix: required
-     *  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
-     */ 
-    Element get_record = OAIXML.createElement(OAIXML.GET_RECORD);
+	*  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
+	*/ 
+    Document doc = this.converter.newDOM();
+    Element get_record = doc.createElement(OAIXML.GET_RECORD);
 
     HashSet<String> valid_strs = new HashSet<String>();
@@ -1067,14 +928,13 @@
     valid_strs.add(OAIXML.METADATA_PREFIX);
 
-    Element req = (Element)GSXML.getChildByTagName(msg, GSXML.REQUEST_ELEM);
-    NodeList params = GSXML.getChildrenByTagName(req, OAIXML.PARAM);
-    HashMap<String, String> param_map = OAIXML.getParamMap(params);    
-    
-    if(!isValidParam(param_map, valid_strs) ||
-        params.getLength() == 0 ||
-        param_map.containsKey(OAIXML.IDENTIFIER) == false ||
-         param_map.containsKey(OAIXML.METADATA_PREFIX) == false ) {
+    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
+    HashMap<String, String> param_map = GSXML.getParamMap(params);    
+    
+    if(!areAllParamsValid(param_map, valid_strs) ||
+       params.getLength() == 0 ||
+       param_map.containsKey(OAIXML.IDENTIFIER) == false ||
+       param_map.containsKey(OAIXML.METADATA_PREFIX) == false ) {
       logger.error("must have the metadataPrefix/identifier parameter.");
-      return getMessage(OAIXML.createErrorElement(OAIXML.BAD_ARGUMENT, ""));
+      return OAIXML.createErrorMessage(OAIXML.BAD_ARGUMENT, "");
     }
     
@@ -1083,54 +943,58 @@
     
     // verify the metadata prefix
-    if (containsMetadataPrefix(prefix) == false) {
+    if (repositorySupportsMetadataPrefix(prefix) == false) {
       logger.error("requested prefix is not found in OAIConfig.xml");
-      return getMessage(OAIXML.createErrorElement(OAIXML.CANNOT_DISSEMINATE_FORMAT, ""));
+      return OAIXML.createErrorMessage(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
     }
 
     // get the names
     String[] strs = splitNames(identifier);
-    if(strs == null || strs.length < 3) {
-      logger.error("identifier is not in the form site:coll:id" + identifier);
-      return getMessage(OAIXML.createErrorElement(OAIXML.ID_DOES_NOT_EXIST, ""));
+    if(strs == null || strs.length < 2) {
+      logger.error("identifier is not in the form coll:id" + identifier);
+      return OAIXML.createErrorMessage(OAIXML.ID_DOES_NOT_EXIST, "");
     }    
-    String name_of_site = strs[0];
-    String coll_name = strs[1];
-    String oid = strs[2];
+    //String name_of_site = strs[0];
+    String coll_name = strs[0];
+    String oid = strs[1];
     
     //re-organize the request element
     // reset the 'to' attribute
-    String verb = req.getAttribute(OAIXML.TO);
-    req.setAttribute(OAIXML.TO, coll_name + "/" + verb);
+    String verb = req.getAttribute(GSXML.TO_ATT);
+    req.setAttribute(GSXML.TO_ATT, coll_name + "/" + verb);
     // reset the identifier element
-    Element param = GSXML.getNamedElement(req, OAIXML.PARAM, OAIXML.NAME, OAIXML.IDENTIFIER);
+    Element param = GSXML.getNamedElement(req, GSXML.PARAM_ELEM, GSXML.NAME_ATT, OAIXML.IDENTIFIER);
     if (param != null) {
-      param.setAttribute(OAIXML.NAME, OAIXML.OID);
-      param.setAttribute(OAIXML.VALUE, oid);
+      param.setAttribute(GSXML.NAME_ATT, OAIXML.OID);
+      param.setAttribute(GSXML.VALUE_ATT, oid);
     }    
 
     //Now send the request to the message router to process
+    Element msg = doc.createElement(GSXML.MESSAGE_ELEM);
+    msg.appendChild(doc.importNode(req, true));
     Node result_node = mr.process(msg);
     return converter.nodeToElement(result_node);
   }
 
-    // See OAIConfig.xml
-    // dynamically works out what the earliestDateStamp is, since it varies by collection
-    // returns this time in *milliseconds*.
-    protected long getEarliestDateStamp(NodeList oai_coll) {
-	//do the earliestDatestamp
-	long earliestDatestamp = System.currentTimeMillis();	
-	int oai_coll_size = oai_coll.getLength();
-	if (oai_coll_size == 0) {
-	    logger.info("returned oai collection list is empty. Setting repository earliestDatestamp to be 1970-01-01.");
-	    earliestDatestamp = 0;
-	}
-	// the earliestDatestamp is now stored as a metadata element in the collection's buildConfig.xml file
-	// we get the earliestDatestamp among the collections
-	for(int i=0; i<oai_coll_size; i++) {
-	    long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_DATESTAMP));
-	    earliestDatestamp = (earliestDatestamp > coll_earliestDatestamp)? coll_earliestDatestamp : earliestDatestamp;
-	}
-
-	return earliestDatestamp*1000; // converting from seconds to milliseconds
-    }
+  // See OAIConfig.xml
+  // dynamically works out what the earliestDateStamp is, since it varies by collection
+  // returns this time in *milliseconds*.
+  protected long getEarliestDateStamp(NodeList oai_coll) {
+    //do the earliestDatestamp
+    long earliestDatestamp = System.currentTimeMillis();	
+    int oai_coll_size = oai_coll.getLength();
+    if (oai_coll_size == 0) {
+      logger.info("returned oai collection list is empty. Setting repository earliestDatestamp to be 1970-01-01.");
+      earliestDatestamp = 0;
+    }
+    // the earliestDatestamp is now stored as a metadata element in the collection's buildConfig.xml file
+    // we get the earliestDatestamp among the collections
+    for(int i=0; i<oai_coll_size; i++) {
+      long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_DATESTAMP));
+      earliestDatestamp = (earliestDatestamp > coll_earliestDatestamp)? coll_earliestDatestamp : earliestDatestamp;
+    }
+
+    return earliestDatestamp*1000; // converting from seconds to milliseconds
+  }
 }
+
+  
