Changeset 31254

Timestamp:

2016-12-20T15:29:56+13:00 (7 years ago)

Author:

davidb

Message:

Experimenting with Lucene lowercase filter

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -12 +12 @@
 import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.Set;
-
 import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONObject;
@@ -20 +18 @@
 import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
 
 public class SolrDocJSON {
@@ -27 +25 @@
                                                              boolean icu_tokenize)
     {
+        boolean lowercase_filter = true;
         
         ArrayList<String> words = new ArrayList<String>();
@@ -39 +38 @@
                     Reader reader = new StringReader(token);
                     
-                    Tokenizer tokenizer = new ICUTokenizer();
-                    tokenizer.setReader(reader);
+                    ICUTokenizer icu_tokenizer = new ICUTokenizer();
+                    icu_tokenizer.setReader(reader);
                         
-                    CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
-
+                    CharTermAttribute charTermAttribute = icu_tokenizer.addAttribute(CharTermAttribute.class);
+
+                    TokenStream token_stream = null;
+                    
+                    if (lowercase_filter) {
+                        token_stream = new LowerCaseFilter(icu_tokenizer);
+                    }
+                    else {
+                        token_stream = icu_tokenizer;
+                    }
+                    
                     try {
-                        tokenizer.reset();
+                        token_stream.reset();
                         
-                        while (tokenizer.incrementToken()) {
+                        while (token_stream.incrementToken()) {
                             String term = charTermAttribute.toString();
                             words.add(term);
                         }
                         
-                        tokenizer.end();
-                        tokenizer.close();
+                        token_stream.end();
+                        token_stream.close();
                     } 
                     catch (IOException e) {