Changeset 32545

Timestamp:

2018-10-29T12:19:46+13:00 (5 years ago)

Author:

kjdon

Message:

changes to query term highlighting. 1. added back in the functionality for getting marked up text back from indexer - useful for solr as it doesn't give you the equivalent terms list, so you don't know what terms it has matched. eg a query for snail will match snail and snails. only do this for text though, not metadata. 2. if the query doesn't give back equivalent terms (solr and lucene), then do case insensitive matching. previously it just did upper and lower case for first letter, but that didn't match eg SNAIL. 3. if the term in the performed query is not in the terms list (eg in the case of solr, query snails, term returned will be snail), but it starts with a term in the term list, then add it to the term list. this helps with a solr query of snails, but not with a query of snail - which matches snails, but snails won't get highlighted. 4. if there is no double quote in the query string, then don't worry about trying to make the phrase structure, just find all the matches of the terms and highlight them. I think thats all...

File:

• main/trunk/greenstone3/src/java/org/greenstone/gsdl3/action/DocumentAction.java (modified) (28 diffs)

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/action/DocumentAction.java

@@ -35 +35 @@
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.io.File;
 import java.io.Serializable;
@@ -541 +542 @@
         GSXML.mergeMetadataLists(the_document, top_doc_node);
 
+        // if we are highlighting query terms, then we also get them highlighted in the metadata
+        
+        HashSet<String> query_term_variants = null;
+        ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy = null;
+        boolean do_highlight_query_terms = highlight_query_terms;
+        int query_terms_status = 0;
+        if (highlight_query_terms) {
+          // lets get the query term equivalents
+          query_term_variants = new HashSet<String>();
+          phrase_query_term_variants_hierarchy = new ArrayList<ArrayList<HashSet<String>>>();
+          if ((query_terms_status = getQueryTermVariants(request, query_term_variants, phrase_query_term_variants_hierarchy)) ==0) {
+            do_highlight_query_terms = false; // we couldn't get the terms
+          }
+        }
+
+        // lets try marking up the metadata with search terms
+        // if the search service doesn't send back <equivTermlist> then we haven't got the term variants. We lower case everything and do case insensitive matching
+        boolean highlight_case_insensitive = false;
+        if (query_terms_status == NO_EQUIV_QUERY_TERMS) {
+          highlight_case_insensitive = true;
+        }
+        if (do_highlight_query_terms) {
+          highlightQueryTermsDOM(doc, the_document, "metadata", query_term_variants, phrase_query_term_variants_hierarchy, highlight_case_insensitive);
+        }
+
         // do we want doc text content? If not, we are done.
         if (!get_text) {
@@ -546 +572 @@
           return result;
         }
-
         
-        HashSet<String> query_term_variants = null;
-        ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy = null;
-        boolean do_highlight_query_terms = highlight_query_terms;
-        if (highlight_query_terms) {
-          // lets get the query term equivalents
-          query_term_variants = new HashSet<String>();
-          phrase_query_term_variants_hierarchy = new ArrayList<ArrayList<HashSet<String>>>();
-          if (!getQueryTermVariants(request, null, /*current_node_id,*/ query_term_variants, phrase_query_term_variants_hierarchy)) {
-            do_highlight_query_terms = false; // we couldn't get the terms
-          }
-        }
-
-        // lets try marking up the metadata with search terms
-        if (do_highlight_query_terms) {
-          highlightQueryTermsDOM(doc, the_document, "metadata", query_term_variants, phrase_query_term_variants_hierarchy);
-        }
-
         // Build a request to obtain some document content
         Element dc_message = doc.createElement(GSXML.MESSAGE_ELEM);
@@ -599 +607 @@
         Element dc_response_doc_list = (Element) GSXML.getNodeByPath(dc_response_message, path);
 
+        boolean get_marked_up_doc_from_query = false;
+        if (do_highlight_query_terms && query_terms_status == NO_EQUIV_QUERY_TERMS) {
+          get_marked_up_doc_from_query = true; // we try to. solr we can, lucene we can't
+        }
+
         if (expand_document)
         {
@@ -606 +619 @@
             {
                 String node_id = ((Element)doc_nodes.item(i)).getAttribute(GSXML.NODE_ID_ATT);
-                //Node content = GSXML.getChildByTagName((Element) dc_response_docs.item(i), GSXML.NODE_CONTENT_ELEM);
                 Node docNode = GSXML.getNamedElement(dc_response_doc_list, "documentNode", GSXML.NODE_ID_ATT, node_id);
                 Node content = GSXML.getChildByTagName(docNode, GSXML.NODE_CONTENT_ELEM);
                 if (content != null)
                 {
-                    if (do_highlight_query_terms)
-                    {
-                      content = highlightQueryTermsElementText(doc, (Element)content, query_term_variants, phrase_query_term_variants_hierarchy);
-                    }
-                    
-                    doc_nodes.item(i).appendChild(doc.importNode(content, true));
-                }
-                //GSXML.mergeMetadataLists(doc_nodes.item(i), dm_response_docs.item(i));
+                  if (do_highlight_query_terms) {
+                    if (get_marked_up_doc_from_query) {
+                      
+                      Element new_content = retrieveHighlightedContent(request, node_id);
+                      
+                      if (new_content == null) {
+                    // we didn't get any text back from the request. assume we won't be able to get it next time either (eg lucene)
+                    get_marked_up_doc_from_query = false;
+                    content = highlightQueryTermsElementText(doc, (Element)content, query_term_variants, phrase_query_term_variants_hierarchy, highlight_case_insensitive);
+                      } else {
+                    content= new_content;
+                      }
+                    } else {
+                      content = highlightQueryTermsElementText(doc, (Element)content, query_term_variants, phrase_query_term_variants_hierarchy, highlight_case_insensitive);
+                    }
+                  }
+                  doc_nodes.item(i).appendChild(doc.importNode(content, true));
+                }
+
             }
             if (has_dummy && document_type.equals(GSXML.DOC_TYPE_SIMPLE)) {
@@ -641 +664 @@
         else
         {
-            //path = GSPath.appendLink(path, GSXML.DOC_NODE_ELEM);
             Element dc_response_doc = (Element) GSXML.getChildByTagName(dc_response_doc_list, GSXML.DOC_NODE_ELEM);
             Element dc_response_doc_content = (Element) GSXML.getChildByTagName(dc_response_doc, GSXML.NODE_CONTENT_ELEM);
-            //Element dc_response_doc_external = (Element) GSXML.getChildByTagName(dc_response_doc, "external");
 
             if (dc_response_doc_content == null)
@@ -651 +672 @@
                 if (dc_response_doc.getAttribute("external").equals("true"))
                 {
-
-                    //if (dc_response_doc_external != null)
-                    //{
                     String href_id = dc_response_doc.getAttribute(GSXML.HREF_ID_ATT);
 
@@ -664 +682 @@
             {
                 dc_response_doc.removeChild(dc_response_doc_content);
-
-                dc_response_doc_content = highlightQueryTermsElementText(doc, (Element)dc_response_doc_content, query_term_variants, phrase_query_term_variants_hierarchy);
+                if (get_marked_up_doc_from_query) {
+                  Element new_content = retrieveHighlightedContent(request, null);
+                  if (new_content == null) {
+                    get_marked_up_doc_from_query = false;
+                    dc_response_doc_content = highlightQueryTermsElementText(doc, (Element)dc_response_doc_content, query_term_variants, phrase_query_term_variants_hierarchy, highlight_case_insensitive);
+                  } else {
+                    
+                    dc_response_doc_content = new_content;
+                  }
+                } else {
+                  dc_response_doc_content = highlightQueryTermsElementText(doc, (Element)dc_response_doc_content, query_term_variants, phrase_query_term_variants_hierarchy, highlight_case_insensitive);
+                }
                 dc_response_doc.appendChild(dc_response_doc.getOwnerDocument().importNode(dc_response_doc_content, true));
             }
@@ -720 +748 @@
 
                     NodeList dummy_children = dummy_node.getChildNodes();
-                    //for (int i=0; i<dummy_children.getLength(); i++) {
                     for (int i = dummy_children.getLength() - 1; i >= 0; i--)
                     {
@@ -1036 +1063 @@
   }
 
+  protected final int NO_QUERY_TERMS = 0;
+  protected final int NO_EQUIV_QUERY_TERMS = 1;
+  protected final int EQUIV_QUERY_TERMS = 2;
   /**
    * this involves a bit of a hack to get the equivalent query terms - has to
    * requery the query service - uses the last selected service name. (if it
-   * ends in query). should this action do the query or should it send a
-   * message to the query action? but that will involve lots of extra stuff. 
+   * ends in query).  
    */
-  protected boolean getQueryTermVariants(Element request, String  current_node_id, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+  protected int getQueryTermVariants(Element request, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
   {
-    Document doc = request.getOwnerDocument();
-    
+    Document doc = XMLConverter.newDOM(); 
+
     // do the query again to get term info 
     Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
-    //logger.error("cgi param list = "+XMLConverter.getPrettyString(cgi_param_list));
     HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
     
@@ -1054 +1082 @@
     if (previous_params == null)
       {
-    //logger.error("no p parms");
-    return false;
+    return NO_QUERY_TERMS;
       }
     String service_name = (String) previous_params.get(GSParams.SERVICE);
     if (service_name == null || !service_name.endsWith("Query"))
       { // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
-    logger.debug("invalid service, not doing highlighting");
-    return false;
+    logger.debug("invalid service "+service_name+", not doing highlighting");
+    return NO_QUERY_TERMS;
       }
 
@@ -1077 +1104 @@
     Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
     GSXML.addParametersToList(query_param_list, service_params);
-    // is this only used for solr??? - do we still want it for solr??
-    // if (current_node_id != null) {
-    //   GSXML.addParameterToList(query_param_list, "hldocOID", current_node_id);
-    // } else {
-    //   GSXML.addParameterToList(query_param_list, "hldocOID", (String) params.get(GSParams.DOCUMENT));
-    // }
     mr_query_request.appendChild(query_param_list);
+
     // do the query
-
     Element mr_query_response = (Element) this.mr.process(mr_query_message);
-
+       
     // find the term lists
     String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
@@ -1094 +1115 @@
       {
     // no term info
-    logger.error("No query term information. xx\n");
-    return false;
+    return  NO_QUERY_TERMS;
       }
-    //    logger.error("query term list info "+XMLConverter.getPrettyString(query_term_list_element));
-    //String content = GSXML.getNodeText(dc_response_doc_content);
-    
-    String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
-    Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
-    
+    
+    int result_code = NO_EQUIV_QUERY_TERMS;
     NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
     if (equivalent_terms_nodelist == null || equivalent_terms_nodelist.getLength() == 0)
       {
+    // if we have no equivalent terms, just add the current terms lower cased and we do case insensitive matching later on
     NodeList terms_nodelist = query_term_list_element.getElementsByTagName("term");
     if (terms_nodelist != null && terms_nodelist.getLength() > 0)
@@ -1112 +1129 @@
           {
         String termValue = ((Element) terms_nodelist.item(i)).getAttribute("name");
-        String termValueU = null;
-        String termValueL = null;
-        
-        if (termValue.length() > 1)
-          {
-            termValueU = termValue.substring(0, 1).toUpperCase() + termValue.substring(1);
-            termValueL = termValue.substring(0, 1).toLowerCase() + termValue.substring(1);
-          }
-        else
-          {
-            termValueU = termValue.substring(0, 1).toUpperCase();
-            termValueL = termValue.substring(0, 1).toLowerCase();
-          }
-        query_term_variants.add(termValueU);
-        query_term_variants.add(termValueL);
+        query_term_variants.add(termValue.toLowerCase());
           }
       }
@@ -1132 +1135 @@
     else
       {
+    result_code = EQUIV_QUERY_TERMS;
     for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++)
       {
@@ -1143 +1147 @@
       }
     
+    String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
+    Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
 
     Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
     String performed_query = GSXML.getNodeText(query_element) + " ";
-
+    logger.debug("performed query="+performed_query);
+
+    boolean has_phrases = false; // if there are no phrases, we don't bother making the phrase variants structure
+    if (performed_query.contains("\"")) {
+      has_phrases = true;
+    }
+    
     ArrayList<HashSet<String>> phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
     int term_start = 0;
     boolean in_term = false;
     boolean in_phrase = false;
-    for (int i = 0; i < performed_query.length(); i++)
-      {
+    for (int i = 0; i < performed_query.length(); i++) {
+      
     char character = performed_query.charAt(i);
     boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
@@ -1168 +1180 @@
         in_term = false;
         String term = performed_query.substring(term_start, i);
-        
-        Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
-        if (term_element != null)
-          {
+        if (has_phrases) {
+          // do the phrase bit
+          HashSet<String> phrase_query_p_term_x_variants = new HashSet<String>();
+          if (result_code == EQUIV_QUERY_TERMS) {
+        Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
+        if (term_element != null) {
+          // might be null for eg TX in [snails]:TX
         
-        HashSet<String> phrase_query_p_term_x_variants = new HashSet<String>();
-        
-        NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
-        if (term_equivalent_terms_nodelist == null || term_equivalent_terms_nodelist.getLength() == 0)
-          {
-            String termValueU = null;
-            String termValueL = null;
-            
-            if (term.length() > 1)
-              {
-            termValueU = term.substring(0, 1).toUpperCase() + term.substring(1);
-            termValueL = term.substring(0, 1).toLowerCase() + term.substring(1);
-              }
-            else
-              {
-            termValueU = term.substring(0, 1).toUpperCase();
-            termValueL = term.substring(0, 1).toLowerCase();
-              }
-            
-            phrase_query_p_term_x_variants.add(termValueU);
-            phrase_query_p_term_x_variants.add(termValueL);
-          }
-        else
-          {
+          NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
+          if (term_equivalent_terms_nodelist != null || term_equivalent_terms_nodelist.getLength() != 0) {
             for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++)
               {
@@ -1207 +1200 @@
               }
           }
+        }
+          } else { // result_code != EQUIV_QUERY_TERMS
+        // we don;t have equivalent term list, so just add the lower cased version in, and we do case-insensitive matching later on
+        if (query_term_variants.contains(term.toLowerCase()) || containsSubString(query_term_variants, term)) {
+          // this handles the case where the user has searched for snails, but term list returns 'snail'
+            phrase_query_p_term_x_variants.add(term.toLowerCase());
+        }
+          }
+          if (phrase_query_p_term_x_variants.size()>0) {
+        // we have found a valid term
         phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
         
@@ -1215 +1218 @@
           }
           }
-      }
+        } // end if has_phrases
+        else {
+          // no phrases so we don't have to do the phrasey stuff. but
+          // we need to check the term against the query term list - if its not in there, check whether its the root of a term.
+          // we want to handle the case where user has queried "snails", the term list returned only has snail, and therefore snails doesn't get highlighted.
+          // but dont want to include eg TX
+          if (result_code == NO_EQUIV_QUERY_TERMS) {
+        if (containsSubString(query_term_variants, term)) {
+          query_term_variants.add(term.toLowerCase());
+        }
+          }
+          
+        } 
+      } // end of in_term... 
     // Watch for phrases (surrounded by quotes)
-    if (character == '\"')
-      {
+    if (character == '\"') {
+      
         // Has a phrase just started?
         if (in_phrase == false)
@@ -1232 +1248 @@
         
         phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
-      }
+    } // if char == "
+    } // for each char in performed query
+  
+    return result_code;
+  }
+
+  protected boolean containsSubString(HashSet<String> query_term_variants, String term) {
+    // hack to filter out TX, TI field names
+    String lc_term = term.toLowerCase();
+    if (query_term_variants.contains(term)) {
+      return false; // or true??
+    }
+    if (term.matches("[A-Z][A-Z][A-Z]?")) {
+      return false;
+    }
+    Iterator i = query_term_variants.iterator();
+    while (i.hasNext()) {
+      String t = (String)i.next();
+      if (term.startsWith(t)) {
+    return true;
       }
-  
-    return true;
+    }
+    return false;
   }
 
-  /** redo the request to get the query terms then highlight them in the text
-   * 
-   */
-  protected Element highlightQueryTermsOld(Element request, String current_node_id, Element dc_response_doc_content)
-    {
-        Document doc = request.getOwnerDocument();
-        
-        // do the query again to get term info 
-        Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
-        HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
-
-        HashMap previous_params = (HashMap) params.get("p");
-        if (previous_params == null)
-        {
-            return dc_response_doc_content;
-        }
-        String service_name = (String) previous_params.get(GSParams.SERVICE);
-        if (service_name == null || !service_name.endsWith("Query"))
-        { // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
-            logger.debug("invalid service, not doing highlighting");
-            return dc_response_doc_content;
-        }
-        String collection = (String) params.get(GSParams.COLLECTION);
-        UserContext userContext = new UserContext(request);
-        String to = GSPath.appendLink(collection, service_name);
-
-        Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
-        Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
-        mr_query_message.appendChild(mr_query_request);
-
-        // paramList
-        HashMap service_params = (HashMap) params.get("s1");
-
-        Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
-        GSXML.addParametersToList(query_param_list, service_params);
-        if (current_node_id != null) {
-          GSXML.addParameterToList(query_param_list, "hldocOID", current_node_id);
-        } else {
-          GSXML.addParameterToList(query_param_list, "hldocOID", (String) params.get(GSParams.DOCUMENT));
-        }
-        mr_query_request.appendChild(query_param_list);
-        // do the query
-        Element mr_query_response = (Element) this.mr.process(mr_query_message);
-        String pathNode = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.NODE_CONTENT_ELEM);
-        Element highlighted_Node = (Element) GSXML.getNodeByPath(mr_query_response, pathNode);
-        // For SOLR, the above query may come back with a nodeContent element, which is the hldocOID section content, with search terms marked up. We send it back to the documnetContentRetrieve service so that resolveTextMacros can be applied, and it can be properly encased in documentNode etc elements
-        if (highlighted_Node != null)
-        {
-            // Build a request to process highlighted text
-          logger.error("highlighted node is not null!!!!");
-          logger.error(XMLConverter.getPrettyString(highlighted_Node));
-            Element hl_message = doc.createElement(GSXML.MESSAGE_ELEM);
-            to = GSPath.appendLink(collection, "DocumentContentRetrieve");
-            Element dc_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
-            hl_message.appendChild(dc_request);
-
-            // Create a parameter list to specify the request parameters - empty for now
-            Element dc_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
-            dc_request.appendChild(dc_param_list);
-
-            // get the content
-            Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
-            dc_request.appendChild(doc_list);
-            Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
-            doc_list.appendChild(current_doc);
-            current_doc.setAttribute(GSXML.NODE_ID_ATT, (String) params.get(GSParams.DOCUMENT));
-            //Append highlighted content to request for processing
-            dc_request.appendChild(doc.importNode(highlighted_Node, true));
-            Element hl_response_message = (Element) this.mr.process(hl_message);
-        
-            //Get results
-            NodeList contentList = hl_response_message.getElementsByTagName(GSXML.NODE_CONTENT_ELEM);
-            Element content = (Element) contentList.item(0);    
-            return content;
-        }
-        String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
-        Element query_term_list_element = (Element) GSXML.getNodeByPath(mr_query_response, path);
-        if (query_term_list_element == null)
-        {
-            // no term info
-            logger.error("No query term information. yy\n");
-            return dc_response_doc_content;
-        }
-
-        String content = GSXML.getNodeText(dc_response_doc_content);
-
-        String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
-        Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
-
-        HashSet<String> query_term_variants = new HashSet<String>();
-        NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
-        if (equivalent_terms_nodelist == null || equivalent_terms_nodelist.getLength() == 0)
-        {
-            NodeList terms_nodelist = query_term_list_element.getElementsByTagName("term");
-            if (terms_nodelist != null && terms_nodelist.getLength() > 0)
-            {
-                for (int i = 0; i < terms_nodelist.getLength(); i++)
-                {
-                    String termValue = ((Element) terms_nodelist.item(i)).getAttribute("name");
-                    String termValueU = null;
-                    String termValueL = null;
-
-                    if (termValue.length() > 1)
-                    {
-                        termValueU = termValue.substring(0, 1).toUpperCase() + termValue.substring(1);
-                        termValueL = termValue.substring(0, 1).toLowerCase() + termValue.substring(1);
-                    }
-                    else
-                    {
-                        termValueU = termValue.substring(0, 1).toUpperCase();
-                        termValueL = termValue.substring(0, 1).toLowerCase();
-                    }
-
-                    query_term_variants.add(termValueU);
-                    query_term_variants.add(termValueL);
-                }
-            }
-        }
-        else
-        {
-            for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++)
-            {
-                Element equivalent_terms_element = (Element) equivalent_terms_nodelist.item(i);
-                String[] equivalent_terms = GSXML.getAttributeValuesFromList(equivalent_terms_element, GSXML.NAME_ATT);
-                for (int j = 0; j < equivalent_terms.length; j++)
-                {
-                    query_term_variants.add(equivalent_terms[j]);
-                }
-            }
-        }
-
-        ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy = new ArrayList<ArrayList<HashSet<String>>>();
-
-        Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
-        String performed_query = GSXML.getNodeText(query_element) + " ";
-
-        ArrayList<HashSet<String>> phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
-        int term_start = 0;
-        boolean in_term = false;
-        boolean in_phrase = false;
-        for (int i = 0; i < performed_query.length(); i++)
-        {
-            char character = performed_query.charAt(i);
-            boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
-
-            // Has a query term just started?
-            if (in_term == false && is_character_letter_or_digit == true)
-            {
-                in_term = true;
-                term_start = i;
-            }
-
-            // Or has a term just finished?
-            else if (in_term == true && is_character_letter_or_digit == false)
-            {
-                in_term = false;
-                String term = performed_query.substring(term_start, i);
-
-                Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
-                if (term_element != null)
-                {
-
-                    HashSet<String> phrase_query_p_term_x_variants = new HashSet<String>();
-
-                    NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
-                    if (term_equivalent_terms_nodelist == null || term_equivalent_terms_nodelist.getLength() == 0)
-                    {
-                        String termValueU = null;
-                        String termValueL = null;
-
-                        if (term.length() > 1)
-                        {
-                            termValueU = term.substring(0, 1).toUpperCase() + term.substring(1);
-                            termValueL = term.substring(0, 1).toLowerCase() + term.substring(1);
-                        }
-                        else
-                        {
-                            termValueU = term.substring(0, 1).toUpperCase();
-                            termValueL = term.substring(0, 1).toLowerCase();
-                        }
-
-                        phrase_query_p_term_x_variants.add(termValueU);
-                        phrase_query_p_term_x_variants.add(termValueL);
-                    }
-                    else
-                    {
-                        for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++)
-                        {
-                            Element term_equivalent_terms_element = (Element) term_equivalent_terms_nodelist.item(j);
-                            String[] term_equivalent_terms = GSXML.getAttributeValuesFromList(term_equivalent_terms_element, GSXML.NAME_ATT);
-                            for (int k = 0; k < term_equivalent_terms.length; k++)
-                            {
-                                phrase_query_p_term_x_variants.add(term_equivalent_terms[k]);
-                            }
-                        }
-                    }
-                    phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
-
-                    if (in_phrase == false)
-                    {
-                        phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
-                        phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
-                    }
-                }
-            }
-            // Watch for phrases (surrounded by quotes)
-            if (character == '\"')
-            {
-                // Has a phrase just started?
-                if (in_phrase == false)
-                {
-                    in_phrase = true;
-                }
-                // Or has a phrase just finished?
-                else if (in_phrase == true)
-                {
-                    in_phrase = false;
-                    phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
-                }
-
-                phrase_query_p_term_variants_list = new ArrayList<HashSet<String>>();
-            }
-        }
-
-        return highlightQueryTermsInternalOrig(doc, content, query_term_variants, phrase_query_term_variants_hierarchy);
-    }
-
+
+  /** retrieve the marked up highlighted section - only works for solr collection */
+  protected Element retrieveHighlightedContent(Element request, String node_id) {
+
+    Document doc = XMLConverter.newDOM();
+
+    // do the query again to get term info 
+    Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+    HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
+    
+    HashMap previous_params = (HashMap) params.get("p");
+    if (previous_params == null)
+      {
+    return null;
+      }
+    String service_name = (String) previous_params.get(GSParams.SERVICE);
+    if (service_name == null || !service_name.endsWith("Query"))
+      { // hack for now - we only do highlighting if we were in a query last - ie not if we were in a browse thingy
+    logger.debug("HL invalid service, not doing highlighting");
+    return null;
+      }
+
+    String collection = (String) params.get(GSParams.COLLECTION);
+    UserContext userContext = new UserContext(request);
+    String to = GSPath.appendLink(collection, service_name);
+    
+    Element mr_query_message = doc.createElement(GSXML.MESSAGE_ELEM);
+    Element mr_query_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+    mr_query_message.appendChild(mr_query_request);
+    
+    // paramList
+    HashMap service_params = (HashMap) params.get("s1");
+
+    // hack in case the user searched on eg titles, but we want highlighting in the text
+    service_params.put("index", "TX");
+    Element query_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+    GSXML.addParametersToList(query_param_list, service_params);
+
+     if (node_id != null) {
+       GSXML.addParameterToList(query_param_list, "hldocOID", node_id);
+     } else {
+       GSXML.addParameterToList(query_param_list, "hldocOID", (String) params.get(GSParams.DOCUMENT));
+     }
+    mr_query_request.appendChild(query_param_list);
+    // do the query
+
+    Element mr_query_response = (Element) this.mr.process(mr_query_message);
+    String pathNode = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.NODE_CONTENT_ELEM);
+    Element highlighted_node = (Element) GSXML.getNodeByPath(mr_query_response, pathNode);
+
+     if (highlighted_node == null) {
+       return null;
+     }
+     // For SOLR, the highlighted node will be a nodeContent element, which is the hldocOID section content, with search terms marked up.
+     //We send it back to the documnetContentRetrieve service so that resolveTextMacros can be applied, and it can be properly encased in documentNode etc elements
+
+     // Build a request to process highlighted text
+     
+     Element hl_message = doc.createElement(GSXML.MESSAGE_ELEM);
+     to = GSPath.appendLink(collection, "DocumentContentRetrieve");
+     Element dc_request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_PROCESS, to, userContext);
+     hl_message.appendChild(dc_request);
+     
+     // Create a parameter list to specify the request parameters - empty for now
+     Element dc_param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
+     dc_request.appendChild(dc_param_list);
+
+     // get the content
+     Element doc_list = doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
+     dc_request.appendChild(doc_list);
+     Element current_doc = doc.createElement(GSXML.DOC_NODE_ELEM);
+     doc_list.appendChild(current_doc);
+     current_doc.setAttribute(GSXML.NODE_ID_ATT, (String) params.get(GSParams.DOCUMENT));
+     //Append highlighted content to request for processing
+     dc_request.appendChild(doc.importNode(highlighted_node, true));
+     Element hl_response_message = (Element) this.mr.process(hl_message);
+     //Get results
+     NodeList contentList = hl_response_message.getElementsByTagName(GSXML.NODE_CONTENT_ELEM);
+     Element content = (Element) contentList.item(0);
+     return content;
+     
+    
+  }
   /** 
    * Highlights query terms in specified elements (whose name is in element_names) text inside top_level_elem
    */
-  protected boolean highlightQueryTermsDOM(Document doc, Element top_level_elem, String element_name, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy) {
-
-    //logger.error("begin highlight DOM "+XMLConverter.getPrettyString(top_level_elem));
+  protected boolean highlightQueryTermsDOM(Document doc, Element top_level_elem, String element_name, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy, boolean case_insensitive) {
+
       NodeList named_elems = top_level_elem.getElementsByTagName(element_name);
       for (int j=named_elems.getLength()-1; j>=0; j--) {
     Element this_elem = (Element)named_elems.item(j);
-    Element replacement_elem = highlightQueryTermsElementText(doc, this_elem, query_term_variants, phrase_query_term_variants_hierarchy);
+    Element replacement_elem = highlightQueryTermsElementText(doc, this_elem, query_term_variants, phrase_query_term_variants_hierarchy, case_insensitive);
     this_elem.getParentNode().replaceChild(replacement_elem, this_elem);
       }
-    
-
-      //logger.error("end highlight DOM "+XMLConverter.getPrettyString(top_level_elem));
     return true;
   }
@@ -1485 +1372 @@
      * Highlights query terms in the text content of an element.
      */
-  private Element highlightQueryTermsElementText(Document doc, Element original_element, /*String content,*/  HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
+  private Element highlightQueryTermsElementText(Document doc, Element original_element,  HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy, boolean case_insensitive)
     {
-      //logger.error("in hl internal, query terms are "+query_term_variants.toString());
       String content = GSXML.getNodeText(original_element);
-      //logger.error("original elem = "+XMLConverter.getPrettyString(original_element));
-      logger.error("highlighting content: "+content);
         // Convert the content string to an array of characters for speed
         char[] content_characters = new char[content.length()];
@@ -1506 +1390 @@
             if (content_characters[i] == '<')
             {
+              // are we currently in a word?
+              if (in_word) {
+                in_word = false;
+                String word = new String(content_characters, word_start, (i - word_start));
+                if (case_insensitive) {
+                  word = word.toLowerCase();
+                }
+                if (query_term_variants.contains(word)) {
+                  // We have found a matching word, so remember its location
+                  word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
+                  // should preceding word matched be set to true/false here??
+                  preceding_word_matched = true;
+                } else {
+                  preceding_word_matched = false;
+                }
+              }
                 inTag = true;
                 continue;
@@ -1512 +1412 @@
             {
                 inTag = false;
+                continue;
             }
             else if (inTag)
@@ -1534 +1435 @@
                 // Check if the word matches any of the query term equivalents
                 String word = new String(content_characters, word_start, (i - word_start));
-                //logger.error("word: "+word);
+                if (case_insensitive) {
+                  word = word.toLowerCase();
+                }
                 if (query_term_variants.contains(word))
                 {
-                  //logger.error("matched");
                     // We have found a matching word, so remember its location
                     word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
@@ -1554 +1456 @@
             // Check if the word matches any of the query term equivalents
             String word = new String(content_characters, word_start, (content_characters.length - word_start));
+            if (case_insensitive) {
+              word = word.toLowerCase();
+            }
             if (query_term_variants.contains(word))
             {
@@ -1561 +1466 @@
         }
 
+        if (word_matches.size() == 0) {
+          // just return a copy of the original element
+          return (Element)doc.importNode(original_element, true);
+          
+        }
+        
         ArrayList<Integer> highlight_start_positions = new ArrayList<Integer>();
         ArrayList<Integer> highlight_end_positions = new ArrayList<Integer>();
-
+        
+        if (phrase_query_term_variants_hierarchy.size() ==0) {
+          for (int i = 0; i < word_matches.size(); i++) {
+            highlight_start_positions.add(new Integer(word_matches.get(i).start_position));
+            highlight_end_positions.add(new Integer(word_matches.get(i).end_position));
+          }
+        }
+        else {
         // Deal with phrases now
         ArrayList<PartialPhraseMatch> partial_phrase_matches = new ArrayList<PartialPhraseMatch>();
@@ -1638 +1556 @@
             }
         }
+        }
 
         // Now add the annotation tags into the document at the correct points
-        //Element content_element = doc.createElement(GSXML.NODE_CONTENT_ELEM);
         Element content_element = (Element)doc.importNode(original_element, false); // just copy the element plus any attributes, but not any children.
         int last_wrote = 0;
@@ -1675 +1593 @@
     }
   
-    private Element highlightQueryTermsInternalOrig(Document doc, String content, HashSet<String> query_term_variants, ArrayList<ArrayList<HashSet<String>>> phrase_query_term_variants_hierarchy)
-    {
-        // Convert the content string to an array of characters for speed
-        char[] content_characters = new char[content.length()];
-        content.getChars(0, content.length(), content_characters, 0);
-
-        // Now skim through the content, identifying word matches
-        ArrayList<WordMatch> word_matches = new ArrayList<WordMatch>();
-        int word_start = 0;
-        boolean in_word = false;
-        boolean preceding_word_matched = false;
-        boolean inTag = false;
-        for (int i = 0; i < content_characters.length; i++)
-        {
-            //We don't want to find words inside HTML tags
-            if (content_characters[i] == '<')
-            {
-                inTag = true;
-                continue;
-            }
-            else if (inTag && content_characters[i] == '>')
-            {
-                inTag = false;
-            }
-            else if (inTag)
-            {
-                continue;
-            }
-
-            boolean is_character_letter_or_digit = Character.isLetterOrDigit(content_characters[i]);
-
-            // Has a word just started?
-            if (in_word == false && is_character_letter_or_digit == true)
-            {
-                in_word = true;
-                word_start = i;
-            }
-
-            // Or has a word just finished?
-            else if (in_word == true && is_character_letter_or_digit == false)
-            {
-                in_word = false;
-
-                // Check if the word matches any of the query term equivalents
-                String word = new String(content_characters, word_start, (i - word_start));
-                if (query_term_variants.contains(word))
-                {
-                    // We have found a matching word, so remember its location
-                    word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
-                    preceding_word_matched = true;
-                }
-                else
-                {
-                    preceding_word_matched = false;
-                }
-            }
-        }
-
-        // Don't forget the last word...
-        if (in_word == true)
-        {
-            // Check if the word matches any of the query term equivalents
-            String word = new String(content_characters, word_start, (content_characters.length - word_start));
-            if (query_term_variants.contains(word))
-            {
-                // We have found a matching word, so remember its location
-                word_matches.add(new WordMatch(word, word_start, content_characters.length, preceding_word_matched));
-            }
-        }
-
-        ArrayList<Integer> highlight_start_positions = new ArrayList<Integer>();
-        ArrayList<Integer> highlight_end_positions = new ArrayList<Integer>();
-
-        // Deal with phrases now
-        ArrayList<PartialPhraseMatch> partial_phrase_matches = new ArrayList<PartialPhraseMatch>();
-        for (int i = 0; i < word_matches.size(); i++)
-        {
-            WordMatch word_match = word_matches.get(i);
-
-            // See if any partial phrase matches are extended by this word
-            if (word_match.preceding_word_matched)
-            {
-                for (int j = partial_phrase_matches.size() - 1; j >= 0; j--)
-                {
-                    PartialPhraseMatch partial_phrase_match = partial_phrase_matches.remove(j);
-                    ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(partial_phrase_match.query_phrase_number);
-                    HashSet phrase_query_p_term_x_variants = (HashSet) phrase_query_p_term_variants_list.get(partial_phrase_match.num_words_matched);
-                    if (phrase_query_p_term_x_variants.contains(word_match.word))
-                    {
-                        partial_phrase_match.num_words_matched++;
-
-                        // Has a complete phrase match occurred?
-                        if (partial_phrase_match.num_words_matched == phrase_query_p_term_variants_list.size())
-                        {
-                            // Check for overlaps by looking at the previous highlight range
-                            if (!highlight_end_positions.isEmpty())
-                            {
-                                int last_highlight_index = highlight_end_positions.size() - 1;
-                                int last_highlight_end = highlight_end_positions.get(last_highlight_index).intValue();
-                                if (last_highlight_end > partial_phrase_match.start_position)
-                                {
-                                    // There is an overlap, so remove the previous phrase match
-                                    int last_highlight_start = highlight_start_positions.remove(last_highlight_index).intValue();
-                                    highlight_end_positions.remove(last_highlight_index);
-                                    partial_phrase_match.start_position = last_highlight_start;
-                                }
-                            }
-
-                            highlight_start_positions.add(new Integer(partial_phrase_match.start_position));
-                            highlight_end_positions.add(new Integer(word_match.end_position));
-                        }
-                        // No, but add the partial match back into the list for next time
-                        else
-                        {
-                            partial_phrase_matches.add(partial_phrase_match);
-                        }
-                    }
-                }
-            }
-            else
-            {
-                partial_phrase_matches.clear();
-            }
-
-            // See if this word is at the start of any of the phrases
-            for (int p = 0; p < phrase_query_term_variants_hierarchy.size(); p++)
-            {
-                ArrayList phrase_query_p_term_variants_list = phrase_query_term_variants_hierarchy.get(p);
-                if (phrase_query_p_term_variants_list.size()>0) {
-                HashSet phrase_query_p_term_1_variants = (HashSet) phrase_query_p_term_variants_list.get(0);
-                if (phrase_query_p_term_1_variants.contains(word_match.word))
-                {
-                    // If this phrase is just one word long, we have a complete match
-                    if (phrase_query_p_term_variants_list.size() == 1)
-                    {
-                        highlight_start_positions.add(new Integer(word_match.start_position));
-                        highlight_end_positions.add(new Integer(word_match.end_position));
-                    }
-                    // Otherwise we have the start of a potential phrase match
-                    else
-                    {
-                        partial_phrase_matches.add(new PartialPhraseMatch(word_match.start_position, p));
-                    }
-                }
-                }
-            }
-        }
-
-        // Now add the annotation tags into the document at the correct points
-        Element content_element = doc.createElement(GSXML.NODE_CONTENT_ELEM);
-
-        int last_wrote = 0;
-        for (int i = 0; i < highlight_start_positions.size(); i++)
-        {
-            int highlight_start = highlight_start_positions.get(i).intValue();
-            int highlight_end = highlight_end_positions.get(i).intValue();
-
-            // Print anything before the highlight range
-            if (last_wrote < highlight_start)
-            {
-                String preceding_text = new String(content_characters, last_wrote, (highlight_start - last_wrote));
-                content_element.appendChild(doc.createTextNode(preceding_text));
-            }
-
-            // Print the highlight text, annotated
-            if (highlight_end > last_wrote)
-            {
-                String highlight_text = new String(content_characters, highlight_start, (highlight_end - highlight_start));
-                Element annotation_element = GSXML.createTextElement(doc, "annotation", highlight_text);
-                annotation_element.setAttribute("type", "query_term");
-                content_element.appendChild(annotation_element);
-                last_wrote = highlight_end;
-            }
-        }
-
-        // Finish off any unwritten text
-        if (last_wrote < content_characters.length)
-        {
-            String remaining_text = new String(content_characters, last_wrote, (content_characters.length - last_wrote));
-            content_element.appendChild(doc.createTextNode(remaining_text));
-        }
-        return content_element;
-    }
 
     static private class WordMatch