Index: main/trunk/greenstone3/src/java/org/greenstone/gsdl3/action/DocumentAction.java
===================================================================
--- main/trunk/greenstone3/src/java/org/greenstone/gsdl3/action/DocumentAction.java	(revision 32504)
+++ main/trunk/greenstone3/src/java/org/greenstone/gsdl3/action/DocumentAction.java	(revision 32505)
@@ -85,5 +85,5 @@
 
 		Element message = GSXML.nodeToElement(message_node);
-		Document doc = XMLConverter.newDOM(); //message.getOwnerDocument();
+		Document doc = XMLConverter.newDOM(); 
 		
 		// the response
@@ -98,4 +98,5 @@
 
 		// just in case there are some that need to get passed to the services
+		// why do we use s0 here and s1 in other places???
 		HashMap service_params = (HashMap) params.get("s0");
 
@@ -167,76 +168,5 @@
 		// are we editing mode? just get the archive document, convert to our internal doc format, and return it
 		if (editing_document) {
-
-		  // call get archive doc
-		  Element dx_message = doc.createElement(GSXML.MESSAGE_ELEM);
-		  String to = "DocXMLGetSection";
-		  Element dx_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
-		  dx_message.appendChild(dx_request);
-		  Element dx_section = doc.createElement(GSXML.DOCXML_SECTION_ELEM);
-		  dx_section.setAttribute(GSXML.NODE_ID_ATT, document_id);
-		  dx_section.setAttribute(GSXML.COLLECTION_ATT, collection);
-		  dx_request.appendChild(dx_section);
-
-		  Element dx_response_message = (Element) this.mr.process(dx_message);
-		  if (processErrorElements(dx_response_message, page_response))
-		    {
-		      return result;
-		    }
-
-		  // get the section out
-		  String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOCXML_SECTION_ELEM);
-		  Element section = (Element) GSXML.getNodeByPath(dx_response_message, path);
-		  if (section == null) {
-		    logger.error("no archive doc returned for "+document_id);
-		    return result;
-		  }
-		  // convert the archive format into the internal format that the page response requires
-
-		  // work out doctype
-		  // NOTE: this will be coming from collection database in index
-		  // the archive file doesn't store this. So we have to assume
-		  // that the doc type will not be changing with any
-		  // modifications happening to archives.
-		  
-		  // if doc type is null, then we need to work it out.
-		  // create a basic doc list containing the current node
-
-		  if (document_type == null) {
-		    Element basic_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
-		    Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
-		    basic_doc_list.appendChild(current_doc);
-		    current_doc.setAttribute(GSXML.NODE_ID_ATT, document_id);
-		    basic_doc_list.appendChild(current_doc);
-		    document_type = getDocumentType(basic_doc_list, collection, userContext, page_response);
-		  }
-		  
-		  if (document_type == null) {
-		      logger.debug("@@@ doctype is null, setting to simple");
-		      document_type = GSXML.DOC_TYPE_SIMPLE;
-		  }		  
-
-		  Element doc_elem = doc.createElement(GSXML.DOCUMENT_ELEM);	 
-		  doc_elem.setAttribute(GSXML.DOC_TYPE_ATT, document_type);
-		  page_response.appendChild(doc_elem);
-
-		  Element transformed_section = transformArchiveToDocument(section);
-		  if (document_type ==  GSXML.DOC_TYPE_SIMPLE) {
-		    // simple doc, only returning a single document node, which is the top level section.
-		    doc_elem.setAttribute(GSXML.NODE_ID_ATT, document_id);
-		    GSXML.mergeElements(doc_elem, transformed_section);
-		    return result;
-		  }
-
-		  // multi sectioned document.
-		  transformed_section.setAttribute(GSXML.NODE_ID_ATT, document_id);		  
-		  // In docEdit mode, we obtain the text from archives, from doc.xml
-		  // Now the transformation has replaced <Section> with <documentNode>
-		  // Need to add nodeID, nodeType and docType attributes to each docNode
-		  // as doc.xml doesn't store that.
-		  insertDocNodeAttributes(transformed_section, document_type, null);		  
-		  doc_elem.appendChild(doc.importNode(transformed_section, true));
-		  logger.debug("dx result = "+XMLConverter.getPrettyString(result));
-
-		  return result;
+		  return getFormattedArchiveDoc(doc, collection, document_id, document_type, result, page_response, userContext);
 		}
 		
@@ -321,4 +251,6 @@
 		
 		the_document.setAttribute(GSXML.DOC_TYPE_ATT, document_type);
+
+		// start getting doc structure
 		
 		// Create a parameter list to specify the required structure information
@@ -482,4 +414,8 @@
 		}
 
+		// end getting doc structure
+
+		// start getting doc metadata
+		
 		// Build a request to obtain some document metadata
 		Element dm_message = doc.createElement(GSXML.MESSAGE_ELEM);
@@ -611,4 +547,22 @@
 		}
 
+		
+		HashSet<String> query_term_variants = null;
+		ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy = null;
+		boolean do_highlight_query_terms = highlight_query_terms;
+		if (highlight_query_terms) {
+		  // lets get the query term equivalents
+		  query_term_variants = new HashSet<String>();
+		  phrase_query_term_variants_hierarchy = new ArrayList<ArrayList<HashSet<String>>>();
+		  if (!getQueryTermVariants(request, null, /*current_node_id,*/ query_term_variants, phrase_query_term_variants_hierarchy)) {
+		    do_highlight_query_terms = false; // we couldn't get the terms
+		  }
+		}
+
+		// lets try marking up the metadata with search terms
+		if (do_highlight_query_terms) {
+		  highlightQueryTermsDOM(doc, the_document, "metadata", query_term_variants, phrase_query_term_variants_hierarchy);
+		}
+
 		// Build a request to obtain some document content
 		Element dc_message = doc.createElement(GSXML.MESSAGE_ELEM);
@@ -657,8 +611,7 @@
 				if (content != null)
 				{
-					if (highlight_query_terms)
+					if (do_highlight_query_terms)
 					{
-					  
-					  content = highlightQueryTerms(request, node_id, (Element) content);
+					  content = highlightQueryTermsElementText(doc, (Element)content, query_term_variants, phrase_query_term_variants_hierarchy);
 					}
 					
@@ -708,9 +661,9 @@
 				return result;
 			}
-			if (highlight_query_terms)
+			if (do_highlight_query_terms)
 			{
 				dc_response_doc.removeChild(dc_response_doc_content);
 
-				dc_response_doc_content = highlightQueryTerms(request, null, dc_response_doc_content);
+				dc_response_doc_content = highlightQueryTermsElementText(doc, (Element)dc_response_doc_content, query_term_variants, phrase_query_term_variants_hierarchy);
 				dc_response_doc.appendChild(dc_response_doc.getOwnerDocument().importNode(dc_response_doc_content, true));
 			}
@@ -803,4 +756,78 @@
 	}
 
+  protected Element getFormattedArchiveDoc(Document doc, String collection, String document_id, String document_type, Element result, Element page_response, UserContext userContext ) {
+    // call get archive doc
+    Element dx_message = doc.createElement(GSXML.MESSAGE_ELEM);
+    String to = "DocXMLGetSection";
+    Element dx_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+    dx_message.appendChild(dx_request);
+    Element dx_section = doc.createElement(GSXML.DOCXML_SECTION_ELEM);
+    dx_section.setAttribute(GSXML.NODE_ID_ATT, document_id);
+    dx_section.setAttribute(GSXML.COLLECTION_ATT, collection);
+    dx_request.appendChild(dx_section);
+
+    Element dx_response_message = (Element) this.mr.process(dx_message);
+    if (processErrorElements(dx_response_message, page_response))
+      {
+	return result;
+      }
+    
+    // get the section out
+    String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.DOCXML_SECTION_ELEM);
+    Element section = (Element) GSXML.getNodeByPath(dx_response_message, path);
+    if (section == null) {
+      logger.error("no archive doc returned for "+document_id);
+      return result;
+    }
+    // convert the archive format into the internal format that the page response requires
+    
+    // work out doctype
+    // NOTE: this will be coming from collection database in index
+    // the archive file doesn't store this. So we have to assume
+    // that the doc type will not be changing with any
+    // modifications happening to archives.
+    
+    // if doc type is null, then we need to work it out.
+    // create a basic doc list containing the current node
+    
+    if (document_type == null) {
+      Element basic_doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+      Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
+      basic_doc_list.appendChild(current_doc);
+      current_doc.setAttribute(GSXML.NODE_ID_ATT, document_id);
+      basic_doc_list.appendChild(current_doc);
+      document_type = getDocumentType(basic_doc_list, collection, userContext, page_response);
+    }
+    
+    if (document_type == null) {
+      logger.debug("@@@ doctype is null, setting to simple");
+      document_type = GSXML.DOC_TYPE_SIMPLE;
+    }		  
+    
+    Element doc_elem = doc.createElement(GSXML.DOCUMENT_ELEM);	 
+    doc_elem.setAttribute(GSXML.DOC_TYPE_ATT, document_type);
+    page_response.appendChild(doc_elem);
+    
+    Element transformed_section = transformArchiveToDocument(section);
+    if (document_type ==  GSXML.DOC_TYPE_SIMPLE) {
+      // simple doc, only returning a single document node, which is the top level section.
+      doc_elem.setAttribute(GSXML.NODE_ID_ATT, document_id);
+      GSXML.mergeElements(doc_elem, transformed_section);
+      return result;
+    }
+    
+    // multi sectioned document.
+    transformed_section.setAttribute(GSXML.NODE_ID_ATT, document_id);		  
+    // In docEdit mode, we obtain the text from archives, from doc.xml
+    // Now the transformation has replaced <Section> with <documentNode>
+    // Need to add nodeID, nodeType and docType attributes to each docNode
+    // as doc.xml doesn't store that.
+    insertDocNodeAttributes(transformed_section, document_type, null);		  
+    doc_elem.appendChild(doc.importNode(transformed_section, true));
+    logger.debug("dx result = "+XMLConverter.getPrettyString(result));
+    
+    return result;
+  }
+  
 	
 	private boolean needSectionContent(HashMap<String, Serializable> params) {
@@ -1009,14 +1036,210 @@
   }
 
-
-	/**
-	 * this involves a bit of a hack to get the equivalent query terms - has to
-	 * requery the query service - uses the last selected service name. (if it
-	 * ends in query). should this action do the query or should it send a
-	 * message to the query action? but that will involve lots of extra stuff.
-	 * also doesn't handle phrases properly - just highlights all the terms
-	 * found in the text.
-	 */
-  protected Element highlightQueryTerms(Element request, String current_node_id, Element dc_response_doc_content)
+  /**
+   * this involves a bit of a hack to get the equivalent query terms - has to
+   * requery the query service - uses the last selected service name. (if it
+   * ends in query). should this action do the query or should it send a
+   * message to the query action? but that will involve lots of extra stuff. 
+   */
+  protected boolean getQueryTermVariants(Element request, String  current_node_id, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+  {
+    Document doc = request.getOwnerDocument();
+    
+    // do the query again to get term info 
+    Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+    //logger.error("cgi param list = "+XMLConverter.getPrettyString(cgi_param_list));
+    HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+    
+    HashMap previous_params = (HashMap) params.get("p");
+    if (previous_params == null)
+      {
+	//logger.error("no p parms");
+	return false;
+      }
+    String service_name = (String) previous_params.get(GSParams.SERVICE);
+    if (service_name == null || !service_name.endsWith("Query"))
+      { // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
+	logger.debug("invalid service, not doing highlighting");
+	return false;
+      }
+
+    String collection = (String) params.get(GSParams.COLLECTION);
+    UserContext userContext = new UserContext(request);
+    String to = GSPath.appendLink(collection, service_name);
+    
+    Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+    mr_query_message.appendChild(mr_query_request);
+    
+    // paramList
+    HashMap service_params = (HashMap) params.get("s1");
+    
+    Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+    GSXML.addParametersToList(query_param_list, service_params);
+    // is this only used for solr??? - do we still want it for solr??
+    // if (current_node_id != null) {
+    //   GSXML.addParameterToList(query_param_list, "hldocOID", current_node_id);
+    // } else {
+    //   GSXML.addParameterToList(query_param_list, "hldocOID", (String) params.get(GSParams.DOCUMENT));
+    // }
+    mr_query_request.appendChild(query_param_list);
+    // do the query
+
+    Element mr_query_response = (Element) this.mr.process(mr_query_message);
+
+    // find the term lists
+    String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
+    Element query_term_list_element = (Element) GSXML.getNodeByPath(mr_query_response, path);
+    if (query_term_list_element == null)
+      {
+	// no term info
+	logger.error("No query term information. xx\n");
+	return false;
+      }
+    //    logger.error("query term list info "+XMLConverter.getPrettyString(query_term_list_element));
+    //String content = GSXML.getNodeText(dc_response_doc_content);
+    
+    String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
+    
+    NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
+    if (equivalent_terms_nodelist == null || equivalent_terms_nodelist.getLength() == 0)
+      {
+	NodeList terms_nodelist = query_term_list_element.getElementsByTagName("term");
+	if (terms_nodelist != null && terms_nodelist.getLength() > 0)
+	  {
+	    for (int i = 0; i < terms_nodelist.getLength(); i++)
+	      {
+		String termValue = ((Element) terms_nodelist.item(i)).getAttribute("name");
+		String termValueU = null;
+		String termValueL = null;
+		
+		if (termValue.length() > 1)
+		  {
+		    termValueU = termValue.substring(0, 1).toUpperCase() + termValue.substring(1);
+		    termValueL = termValue.substring(0, 1).toLowerCase() + termValue.substring(1);
+		  }
+		else
+		  {
+		    termValueU = termValue.substring(0, 1).toUpperCase();
+		    termValueL = termValue.substring(0, 1).toLowerCase();
+		  }
+		query_term_variants.add(termValueU);
+		query_term_variants.add(termValueL);
+	      }
+	  }
+      }
+    else
+      {
+	for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++)
+	  {
+	    Element equivalent_terms_element = (Element) equivalent_terms_nodelist.item(i);
+	    String[] equivalent_terms = GSXML.getAttributeValuesFromList(equivalent_terms_element, GSXML.NAME_ATT);
+	    for (int j = 0; j < equivalent_terms.length; j++)
+	      {
+		query_term_variants.add(equivalent_terms[j]);
+	      }
+	  }
+      }
+    
+
+    Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
+    String performed_query = GSXML.getNodeText(query_element) + " ";
+
+    ArrayList<HashSet<String>> phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+    int term_start = 0;
+    boolean in_term = false;
+    boolean in_phrase = false;
+    for (int i = 0; i < performed_query.length(); i++)
+      {
+	char character = performed_query.charAt(i);
+	boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
+	
+	// Has a query term just started?
+	if (in_term == false && is_character_letter_or_digit == true)
+	  {
+	    in_term = true;
+	    term_start = i;
+	  }
+	
+	// Or has a term just finished?
+	else if (in_term == true && is_character_letter_or_digit == false)
+	  {
+	    in_term = false;
+	    String term = performed_query.substring(term_start, i);
+	    
+	    Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
+	    if (term_element != null)
+	      {
+		
+		HashSet<String> phrase_query_p_term_x_variants = new HashSet<String>();
+		
+		NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
+		if (term_equivalent_terms_nodelist == null || term_equivalent_terms_nodelist.getLength() == 0)
+		  {
+		    String termValueU = null;
+		    String termValueL = null;
+		    
+		    if (term.length() > 1)
+		      {
+			termValueU = term.substring(0, 1).toUpperCase() + term.substring(1);
+			termValueL = term.substring(0, 1).toLowerCase() + term.substring(1);
+		      }
+		    else
+		      {
+			termValueU = term.substring(0, 1).toUpperCase();
+			termValueL = term.substring(0, 1).toLowerCase();
+		      }
+		    
+		    phrase_query_p_term_x_variants.add(termValueU);
+		    phrase_query_p_term_x_variants.add(termValueL);
+		  }
+		else
+		  {
+		    for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++)
+		      {
+			Element term_equivalent_terms_element = (Element) term_equivalent_terms_nodelist.item(j);
+			String[] term_equivalent_terms = GSXML.getAttributeValuesFromList(term_equivalent_terms_element, GSXML.NAME_ATT);
+			for (int k = 0; k < term_equivalent_terms.length; k++)
+			  {
+			    phrase_query_p_term_x_variants.add(term_equivalent_terms[k]);
+			  }
+		      }
+		  }
+		phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
+		
+		if (in_phrase == false)
+		  {
+		    phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+		    phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+		  }
+	      }
+	  }
+	// Watch for phrases (surrounded by quotes)
+	if (character == '\"')
+	  {
+	    // Has a phrase just started?
+	    if (in_phrase == false)
+	      {
+		in_phrase = true;
+	      }
+	    // Or has a phrase just finished?
+	    else if (in_phrase == true)
+	      {
+		in_phrase = false;
+		phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+	      }
+	    
+	    phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
+	  }
+      }
+  
+    return true;
+  }
+
+  /** redo the request to get the query terms then highlight them in the text
+   * 
+   */
+  protected Element highlightQueryTermsOld(Element request, String current_node_id, Element dc_response_doc_content)
 	{
 		Document doc = request.getOwnerDocument();
@@ -1064,5 +1287,6 @@
 		{
 			// Build a request to process highlighted text
-			
+		  logger.error("highlighted node is not null!!!!");
+		  logger.error(XMLConverter.getPrettyString(highlighted_Node));
 			Element hl_message = doc.createElement(GSXML.MESSAGE_ELEM);
 			to = GSPath.appendLink(collection, "DocumentContentRetrieve");
@@ -1094,5 +1318,5 @@
 		{
 			// no term info
-			logger.error("No query term information.\n");
+			logger.error("No query term information. yy\n");
 			return dc_response_doc_content;
 		}
@@ -1238,12 +1462,33 @@
 		}
 
-		return highlightQueryTermsInternal(doc, content, query_term_variants, phrase_query_term_variants_hierarchy);
+		return highlightQueryTermsInternalOrig(doc, content, query_term_variants, phrase_query_term_variants_hierarchy);
 	}
 
+  /** 
+   * Highlights query terms in specified elements (whose name is in element_names) text inside top_level_elem
+   */
+  protected boolean highlightQueryTermsDOM(Document doc, Element top_level_elem, String element_name, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy) {
+
+    //logger.error("begin highlight DOM "+XMLConverter.getPrettyString(top_level_elem));
+      NodeList named_elems = top_level_elem.getElementsByTagName(element_name);
+      for (int j=named_elems.getLength()-1; j>=0; j--) {
+	Element this_elem = (Element)named_elems.item(j);
+	Element replacement_elem = highlightQueryTermsElementText(doc, this_elem, query_term_variants, phrase_query_term_variants_hierarchy);
+	this_elem.getParentNode().replaceChild(replacement_elem, this_elem);
+      }
+    
+
+      //logger.error("end highlight DOM "+XMLConverter.getPrettyString(top_level_elem));
+    return true;
+  }
 	/**
-	 * Highlights query terms in a piece of text.
+	 * Highlights query terms in the text content of an element.
 	 */
-	private Element highlightQueryTermsInternal(Document doc, String content, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+  private Element highlightQueryTermsElementText(Document doc, Element original_element, /*String content,*/  HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
 	{
+	  //logger.error("in hl internal, query terms are "+query_term_variants.toString());
+	  String content = GSXML.getNodeText(original_element);
+	  //logger.error("original elem = "+XMLConverter.getPrettyString(original_element));
+	  logger.error("highlighting content: "+content);
 		// Convert the content string to an array of characters for speed
 		char[] content_characters = new char[content.length()];
@@ -1289,6 +1534,8 @@
 				// Check if the word matches any of the query term equivalents
 				String word = new String(content_characters, word_start, (i - word_start));
+				//logger.error("word: "+word);
 				if (query_term_variants.contains(word))
 				{
+				  //logger.error("matched");
 					// We have found a matching word, so remember its location
 					word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
@@ -1393,4 +1640,188 @@
 
 		// Now add the annotation tags into the document at the correct points
+		//Element content_element = doc.createElement(GSXML.NODE_CONTENT_ELEM);
+		Element content_element = (Element)doc.importNode(original_element, false); // just copy the element plus any attributes, but not any children.
+		int last_wrote = 0;
+		for (int i = 0; i < highlight_start_positions.size(); i++)
+		{
+			int highlight_start = highlight_start_positions.get(i).intValue();
+			int highlight_end = highlight_end_positions.get(i).intValue();
+
+			// Print anything before the highlight range
+			if (last_wrote < highlight_start)
+			{
+				String preceding_text = new String(content_characters, last_wrote, (highlight_start - last_wrote));
+				content_element.appendChild(doc.createTextNode(preceding_text));
+			}
+
+			// Print the highlight text, annotated
+			if (highlight_end > last_wrote)
+			{
+				String highlight_text = new String(content_characters, highlight_start, (highlight_end - highlight_start));
+				Element annotation_element = GSXML.createTextElement(doc, "annotation", highlight_text);
+				annotation_element.setAttribute("type", "query_term");
+				content_element.appendChild(annotation_element);
+				last_wrote = highlight_end;
+			}
+		}
+
+		// Finish off any unwritten text
+		if (last_wrote < content_characters.length)
+		{
+			String remaining_text = new String(content_characters, last_wrote, (content_characters.length - last_wrote));
+			content_element.appendChild(doc.createTextNode(remaining_text));
+		}
+		return content_element;
+	}
+  
+	private Element highlightQueryTermsInternalOrig(Document doc, String content, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+	{
+		// Convert the content string to an array of characters for speed
+		char[] content_characters = new char[content.length()];
+		content.getChars(0, content.length(), content_characters, 0);
+
+		// Now skim through the content, identifying word matches
+		ArrayList<WordMatch> word_matches = new ArrayList<WordMatch>();
+		int word_start = 0;
+		boolean in_word = false;
+		boolean preceding_word_matched = false;
+		boolean inTag = false;
+		for (int i = 0; i < content_characters.length; i++)
+		{
+			//We don't want to find words inside HTML tags
+			if (content_characters[i] == '<')
+			{
+				inTag = true;
+				continue;
+			}
+			else if (inTag && content_characters[i] == '>')
+			{
+				inTag = false;
+			}
+			else if (inTag)
+			{
+				continue;
+			}
+
+			boolean is_character_letter_or_digit = Character.isLetterOrDigit(content_characters[i]);
+
+			// Has a word just started?
+			if (in_word == false && is_character_letter_or_digit == true)
+			{
+				in_word = true;
+				word_start = i;
+			}
+
+			// Or has a word just finished?
+			else if (in_word == true && is_character_letter_or_digit == false)
+			{
+				in_word = false;
+
+				// Check if the word matches any of the query term equivalents
+				String word = new String(content_characters, word_start, (i - word_start));
+				if (query_term_variants.contains(word))
+				{
+					// We have found a matching word, so remember its location
+					word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
+					preceding_word_matched = true;
+				}
+				else
+				{
+					preceding_word_matched = false;
+				}
+			}
+		}
+
+		// Don't forget the last word...
+		if (in_word == true)
+		{
+			// Check if the word matches any of the query term equivalents
+			String word = new String(content_characters, word_start, (content_characters.length - word_start));
+			if (query_term_variants.contains(word))
+			{
+				// We have found a matching word, so remember its location
+				word_matches.add(new WordMatch(word, word_start, content_characters.length, preceding_word_matched));
+			}
+		}
+
+		ArrayList<Integer> highlight_start_positions = new ArrayList<Integer>();
+		ArrayList<Integer> highlight_end_positions = new ArrayList<Integer>();
+
+		// Deal with phrases now
+		ArrayList<PartialPhraseMatch> partial_phrase_matches = new ArrayList<PartialPhraseMatch>();
+		for (int i = 0; i < word_matches.size(); i++)
+		{
+			WordMatch word_match = word_matches.get(i);
+
+			// See if any partial phrase matches are extended by this word
+			if (word_match.preceding_word_matched)
+			{
+				for (int j = partial_phrase_matches.size() - 1; j >= 0; j--)
+				{
+					PartialPhraseMatch partial_phrase_match = partial_phrase_matches.remove(j);
+					ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(partial_phrase_match.query_phrase_number);
+					HashSet phrase_query_p_term_x_variants = (HashSet) phrase_query_p_term_variants_list.get(partial_phrase_match.num_words_matched);
+					if (phrase_query_p_term_x_variants.contains(word_match.word))
+					{
+						partial_phrase_match.num_words_matched++;
+
+						// Has a complete phrase match occurred?
+						if (partial_phrase_match.num_words_matched == phrase_query_p_term_variants_list.size())
+						{
+							// Check for overlaps by looking at the previous highlight range
+							if (!highlight_end_positions.isEmpty())
+							{
+								int last_highlight_index = highlight_end_positions.size() - 1;
+								int last_highlight_end = highlight_end_positions.get(last_highlight_index).intValue();
+								if (last_highlight_end > partial_phrase_match.start_position)
+								{
+									// There is an overlap, so remove the previous phrase match
+									int last_highlight_start = highlight_start_positions.remove(last_highlight_index).intValue();
+									highlight_end_positions.remove(last_highlight_index);
+									partial_phrase_match.start_position = last_highlight_start;
+								}
+							}
+
+							highlight_start_positions.add(new Integer(partial_phrase_match.start_position));
+							highlight_end_positions.add(new Integer(word_match.end_position));
+						}
+						// No, but add the partial match back into the list for next time
+						else
+						{
+							partial_phrase_matches.add(partial_phrase_match);
+						}
+					}
+				}
+			}
+			else
+			{
+				partial_phrase_matches.clear();
+			}
+
+			// See if this word is at the start of any of the phrases
+			for (int p = 0; p < phrase_query_term_variants_hierarchy.size(); p++)
+			{
+				ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(p);
+				if (phrase_query_p_term_variants_list.size()>0) {
+				HashSet phrase_query_p_term_1_variants = (HashSet) phrase_query_p_term_variants_list.get(0);
+				if (phrase_query_p_term_1_variants.contains(word_match.word))
+				{
+					// If this phrase is just one word long, we have a complete match
+					if (phrase_query_p_term_variants_list.size() == 1)
+					{
+						highlight_start_positions.add(new Integer(word_match.start_position));
+						highlight_end_positions.add(new Integer(word_match.end_position));
+					}
+					// Otherwise we have the start of a potential phrase match
+					else
+					{
+						partial_phrase_matches.add(new PartialPhraseMatch(word_match.start_position, p));
+					}
+				}
+				}
+			}
+		}
+
+		// Now add the annotation tags into the document at the correct points
 		Element content_element = doc.createElement(GSXML.NODE_CONTENT_ELEM);
 
