Changeset 8731

Timestamp:

2004-12-03T13:09:47+13:00 (19 years ago)

Author:

mdewsnip

Message:

Much more advanced query term highlighting. Supports query term highlighting and query phrase highlighting, with all permutations of case folding and stemming.

File:

• trunk/gsdl3/src/java/org/greenstone/gsdl3/action/DocumentAction.java (modified) (4 diffs)

trunk/gsdl3/src/java/org/greenstone/gsdl3/action/DocumentAction.java

@@ -31 +31 @@
 
 // General Java classes
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -48 +49 @@
      * type services (enrich) will be offered to the user as well */
     protected static final boolean provide_annotations = false; //true;
+
 
     public Element process (Element message)
@@ -540 +542 @@
     
     String path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.TERM_ELEM+GSXML.LIST_MODIFIER);
-    Element query_term_info_list = (Element) GSXML.getNodeByPath(mr_query_response, path);
-    if (query_term_info_list == null) {
+    Element query_term_list_element = (Element) GSXML.getNodeByPath(mr_query_response, path);
+    if (query_term_list_element == null) {
         // no term info
         System.err.println("DocumentAction: Warning: No query term information.\n");
@@ -547 +549 @@
     } 
 
-    NodeList equivs = query_term_info_list.getElementsByTagName("equivTermList");
-    HashSet all_terms = new HashSet();
-    for (int i=0; i<equivs.getLength(); i++) {
-        
-        // get the terms
-        String [] terms = GSXML.getAttributeValuesFromList((Element)equivs.item(i), GSXML.NAME_ATT);
-        for (int j=0; j<terms.length; j++) {
-        
-        all_terms.add(terms[j]);
-        }
-    }
-
-    Element new_content_elem = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
-
     String content = GSXML.getNodeText(dc_response_doc_content);
 
-    StringBuffer temp = new StringBuffer();
-    StringBuffer temp_content = new StringBuffer();
-
-    for (int i=0; i<content.length(); i++) {
-        char c = content.charAt(i);
-        if (Character.isLetterOrDigit(c)) {
-        // not word boundary
-        temp.append(c);
-        } else {
-        // word boundary
-        // add the last word if there was one
-        if (temp.length()>0) {
-            if (all_terms.contains(temp.toString())) {
-            //if there is anything already present in temp_content, add it as a text node
-            Text t = this.doc.createTextNode(temp_content.toString());
-            new_content_elem.appendChild(t);
-            temp_content.delete(0, temp_content.length());
-            Element annot = GSXML.createTextElement(this.doc, "annotation", temp.toString());
-            annot.setAttribute("type", "query_term");
-            new_content_elem.appendChild(annot);
-            //new_content.append("<annotation type='query_term'>"+temp+"</annotation>");
-            } else {
-            temp_content.append(temp);
+    String metadata_path = GSPath.appendLink(GSXML.RESPONSE_ELEM, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
+    Element metadata_list = (Element) GSXML.getNodeByPath(mr_query_response, metadata_path);
+
+    HashSet query_term_variants = new HashSet();
+    NodeList equivalent_terms_nodelist = query_term_list_element.getElementsByTagName("equivTermList");
+    for (int i = 0; i < equivalent_terms_nodelist.getLength(); i++) {
+        Element equivalent_terms_element = (Element) equivalent_terms_nodelist.item(i);
+        String[] equivalent_terms = GSXML.getAttributeValuesFromList(equivalent_terms_element, GSXML.NAME_ATT);
+        for (int j = 0; j < equivalent_terms.length; j++) {
+        System.err.println("Adding query term variant: " + equivalent_terms[j]);
+        query_term_variants.add(equivalent_terms[j]);
+        }
+    }
+
+    ArrayList phrase_query_term_variants_hierarchy = new ArrayList();
+
+    Element query_element = GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "query");
+    String performed_query = GSXML.getNodeText(query_element) + " ";
+
+    ArrayList phrase_query_p_term_variants_list = new ArrayList();
+    int term_start = 0;
+    boolean in_term = false;
+    boolean in_phrase = false;
+    for (int i = 0; i < performed_query.length(); i++) {
+        char character = performed_query.charAt(i);
+        boolean is_character_letter_or_digit = Character.isLetterOrDigit(character);
+
+        // Has a query term just started?
+        if (in_term == false && is_character_letter_or_digit == true) {
+        in_term = true;
+        term_start = i;
+        }
+
+        // Or has a term just finished?
+        else if (in_term == true && is_character_letter_or_digit == false) {
+        in_term = false;
+        String term = performed_query.substring(term_start, i);
+        System.err.println("Term: " + term);
+
+        HashSet phrase_query_p_term_x_variants = new HashSet();
+        Element term_element = GSXML.getNamedElement(query_term_list_element, GSXML.TERM_ELEM, GSXML.NAME_ATT, term);
+        NodeList term_equivalent_terms_nodelist = term_element.getElementsByTagName("equivTermList");
+        for (int j = 0; j < term_equivalent_terms_nodelist.getLength(); j++) {
+            Element term_equivalent_terms_element = (Element) term_equivalent_terms_nodelist.item(j);
+            String[] term_equivalent_terms = GSXML.getAttributeValuesFromList(term_equivalent_terms_element, GSXML.NAME_ATT);
+            for (int k = 0; k < term_equivalent_terms.length; k++) {
+            System.err.println("Adding query term variant: " + term_equivalent_terms[k]);
+            phrase_query_p_term_x_variants.add(term_equivalent_terms[k]);
             }
-            temp.delete(0, temp.length());
-        }
-        if (c=='<') {
-            temp_content.append(c);
-            i++;
-            // skip over html
-            while (i<content.length() && content.charAt(i)!='>') {
-            temp_content.append(content.charAt(i));
-            i++;
+        }
+        phrase_query_p_term_variants_list.add(phrase_query_p_term_x_variants);
+
+        if (in_phrase == false) {
+            phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+            phrase_query_p_term_variants_list = new ArrayList();
+        }
+        }
+
+        // Watch for phrases (surrounded by quotes)
+        if (character == '\"') {
+        // Has a phrase just started?
+        if (in_phrase == false) {
+            in_phrase = true;
+        }
+        // Or has a phrase just finished?
+        else if (in_phrase == true) {
+            in_phrase = false;
+            phrase_query_term_variants_hierarchy.add(phrase_query_p_term_variants_list);
+        }
+
+        phrase_query_p_term_variants_list = new ArrayList();
+        }
+    }
+
+    return highlightQueryTermsInternal(content, query_term_variants, phrase_query_term_variants_hierarchy);
+    }
+
+
+    /**
+     * Highlights query terms in a piece of text.
+     */
+    private Element highlightQueryTermsInternal(String content, HashSet query_term_variants, ArrayList phrase_query_term_variants_hierarchy)
+    {
+    // Convert the content string to an array of characters for speed
+    char[] content_characters = new char[content.length()];
+    content.getChars(0, content.length(), content_characters, 0);
+
+    // Now skim through the content, identifying word matches
+    ArrayList word_matches = new ArrayList();
+    int word_start = 0;
+    boolean in_word = false;
+    boolean preceding_word_matched = false;
+    for (int i = 0; i < content_characters.length; i++) {
+        boolean is_character_letter_or_digit = Character.isLetterOrDigit(content_characters[i]);
+
+        // Has a word just started?
+        if (in_word == false && is_character_letter_or_digit == true) {
+        in_word = true;
+        word_start = i;
+        }
+
+        // Or has a word just finished?
+        else if (in_word == true && is_character_letter_or_digit == false) {
+        in_word = false;
+
+        // Check if the word matches any of the query term equivalents
+        String word = new String(content_characters, word_start, (i - word_start));
+        if (query_term_variants.contains(word)) {
+            // We have found a matching word, so remember its location
+            word_matches.add(new WordMatch(word, word_start, i, preceding_word_matched));
+            preceding_word_matched = true;
+        }
+        else {
+            preceding_word_matched = false;
+        }
+        }
+    }
+
+    // Don't forget the last word...
+    if (in_word == true) {
+        // Check if the word matches any of the query term equivalents
+        String word = new String(content_characters, word_start, (content_characters.length - word_start));
+        if (query_term_variants.contains(word)) {
+        // We have found a matching word, so remember its location
+        word_matches.add(new WordMatch(word, word_start, content_characters.length, preceding_word_matched));
+        }
+    }
+
+    ArrayList highlight_start_positions = new ArrayList();
+    ArrayList highlight_end_positions = new ArrayList();
+
+    // Deal with phrases now
+    ArrayList partial_phrase_matches = new ArrayList();
+    for (int i = 0; i < word_matches.size(); i++) {
+        WordMatch word_match = (WordMatch) word_matches.get(i);
+
+        // See if any partial phrase matches are extended by this word
+        if (word_match.preceding_word_matched) {
+        for (int j = partial_phrase_matches.size() - 1; j >= 0; j--) {
+            PartialPhraseMatch partial_phrase_match = (PartialPhraseMatch) partial_phrase_matches.remove(j);
+            ArrayList phrase_query_p_term_variants_list = (ArrayList) phrase_query_term_variants_hierarchy.get(partial_phrase_match.query_phrase_number);
+            HashSet phrase_query_p_term_x_variants = (HashSet) phrase_query_p_term_variants_list.get(partial_phrase_match.num_words_matched);
+            if (phrase_query_p_term_x_variants.contains(word_match.word)) {
+            partial_phrase_match.num_words_matched++;
+
+            // Has a complete phrase match occurred?
+            if (partial_phrase_match.num_words_matched == phrase_query_p_term_variants_list.size()) {
+                // Check for overlaps by looking at the previous highlight range
+                if (!highlight_end_positions.isEmpty()) {
+                int last_highlight_index = highlight_end_positions.size() - 1;
+                int last_highlight_end = ((Integer) highlight_end_positions.get(last_highlight_index)).intValue();
+                if (last_highlight_end > partial_phrase_match.start_position) {
+                    // There is an overlap, so remove the previous phrase match
+                    int last_highlight_start = ((Integer) highlight_start_positions.remove(last_highlight_index)).intValue();
+                    highlight_end_positions.remove(last_highlight_index);
+                    partial_phrase_match.start_position = last_highlight_start;
+                }
+                }
+
+                highlight_start_positions.add(new Integer(partial_phrase_match.start_position));
+                highlight_end_positions.add(new Integer(word_match.end_position));
+            }
+            // No, but add the partial match back into the list for next time
+            else {
+                partial_phrase_matches.add(partial_phrase_match);
+            }
+            }
+        }
+        }
+        else {
+        partial_phrase_matches.clear();
+        }
+
+        // See if this word is at the start of any of the phrases
+        for (int p = 0; p < phrase_query_term_variants_hierarchy.size(); p++) {
+        ArrayList phrase_query_p_term_variants_list = (ArrayList) phrase_query_term_variants_hierarchy.get(p);
+        HashSet phrase_query_p_term_1_variants = (HashSet) phrase_query_p_term_variants_list.get(0);
+        if (phrase_query_p_term_1_variants.contains(word_match.word)) {
+            // If this phrase is just one word long, we have a complete match
+            if (phrase_query_p_term_variants_list.size() == 1) {
+            highlight_start_positions.add(new Integer(word_match.start_position));
+            highlight_end_positions.add(new Integer(word_match.end_position));
             }
-            temp_content.append(content.charAt(i));
-            //temp_content.append(GSXML.xmlSafe(temp.toString()));
-            //temp.delete(0, temp.length());
-            
-        } else {
-            temp_content.append(c);
-        }
-        }
-    }
-    // append anything left of temp_content and temp
-    Text t = this.doc.createTextNode(temp_content.toString());
-    new_content_elem.appendChild(t);
-
-    if (temp.length() > 0) {
-        Element annot = GSXML.createTextElement(this.doc, "annotation", temp.toString());
-        annot.setAttribute("type", "query_term");
-        new_content_elem.appendChild(annot);
-    }
-    //String content_string = "<nodeContent>"+new_content.toString()+"</nodeContent>";
-    //Element content_elem = this.converter.getDOM(content_string).getDocumentElement();
-    return new_content_elem;
+            // Otherwise we have the start of a potential phrase match
+            else {
+            partial_phrase_matches.add(new PartialPhraseMatch(word_match.start_position, p));
+            }
+        }
+        }
+    }
+
+    // Now add the annotation tags into the document at the correct points
+    Element content_element = this.doc.createElement(GSXML.NODE_CONTENT_ELEM);
+
+    int last_wrote = 0;
+    for (int i = 0; i < highlight_start_positions.size(); i++) {
+        int highlight_start = ((Integer) highlight_start_positions.get(i)).intValue();
+        int highlight_end = ((Integer) highlight_end_positions.get(i)).intValue();
+
+        // Print anything before the highlight range
+        if (last_wrote < highlight_start) {
+        String preceding_text = new String(content_characters, last_wrote, (highlight_start - last_wrote));
+        // System.err.print(preceding_text);
+        content_element.appendChild(this.doc.createTextNode(preceding_text));
+        }
+
+        // Print the highlight text, annotated
+        if (highlight_end > last_wrote) {
+        String highlight_text = new String(content_characters, highlight_start, (highlight_end - highlight_start));
+        // System.err.print("|" + highlight_text + "|");
+        Element annotation_element = GSXML.createTextElement(this.doc, "annotation", highlight_text);
+        annotation_element.setAttribute("type", "query_term");
+        content_element.appendChild(annotation_element);
+        last_wrote = highlight_end;
+        }
+    }
+
+    // Finish off any unwritten text
+    if (last_wrote < content_characters.length) {
+        String remaining_text = new String(content_characters, last_wrote, (content_characters.length - last_wrote));
+        // System.err.print(remaining_text);
+        content_element.appendChild(this.doc.createTextNode(remaining_text));
+    }
+
+    return content_element;
+    }
+
+
+    static private class WordMatch
+    {
+    public String word;
+    public int start_position;
+    public int end_position;
+    public boolean preceding_word_matched;
+
+    public WordMatch(String word, int start_position, int end_position, boolean preceding_word_matched)
+    {
+        this.word = word;
+        this.start_position = start_position;
+        this.end_position = end_position;
+        this.preceding_word_matched = preceding_word_matched;
+    }
+    }
+
+
+    static private class PartialPhraseMatch
+    {
+    public int start_position;
+    public int query_phrase_number;
+    public int num_words_matched;
+
+    public PartialPhraseMatch(int start_position, int query_phrase_number)
+    {
+        this.start_position = start_position;
+        this.query_phrase_number = query_phrase_number;
+        this.num_words_matched = 1;
+    }
     }
 }