Changeset 31243 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Timestamp:

2016-12-17T17:25:08+13:00 (7 years ago)

Author:

davidb

Message:

Experimenting with Lucene/Solr's ICU tokenizer

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• pom.xml (modified) (2 diffs)
• src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/pom.xml

@@ -51 +51 @@
       <version>20.0</version>
     </dependency>
-    
+
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analyzers-icu</artifactId>
+      <version>6.3.0</version>
+    </dependency>
+
   </dependencies>
 
@@ -64 +70 @@
        <source>1.8</source>
        <target>1.8</target>
+
      </configuration>
        </plugin>

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -6 +6 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Set;
 
 import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONObject;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 
 public class SolrDocJSON {
@@ -18 +25 @@
     protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
         {
-            StringBuilder sb = new StringBuilder();
-    
+            boolean solr_icu_tokenize = true;
+        
+            ArrayList<String> tokens = new ArrayList<String>();
+            
             Iterator<String> token_iter = ef_token_pos_count.keys();
             
+            while (token_iter.hasNext()) {
+                String token = token_iter.next();
+
+                if (solr_icu_tokenize == true) {
+                    Reader reader = new StringReader(token);
+                    
+                    Tokenizer tokenizer = new ICUTokenizer();
+                    tokenizer.setReader(reader);
+                        
+                    //TokenStream tokenStream = analyzer.tokenStream(fieldName, reader);
+                    //OffsetAttribute offsetAttribute = tokenizer.addAttribute(OffsetAttribute.class);
+                    CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
+
+                    try {
+                        tokenizer.reset();
+                        
+                        while (tokenizer.incrementToken()) {
+                            //int startOffset = offsetAttribute.startOffset();
+                            //int endOffset = offsetAttribute.endOffset();
+                            String term = charTermAttribute.toString();
+                            tokens.add(term);
+                        }
+                        
+                        tokenizer.close();
+                    } 
+                    catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                }
+                else {
+                    tokens.add(token);
+                }
+            }
+            
+            StringBuilder sb = new StringBuilder();
+            
             if (whitelist_bloomfilter == null) {
 
-                while (token_iter.hasNext()) {
-                    String token = token_iter.next();
+                boolean first_append = true;
+                
+                for (int i=0; i<tokens.size(); i++) {
+                    String token = tokens.get(i);
+                                    
+                    if (!first_append) {
+                        sb.append(" ");
+                    }
+                    else {
+                        first_append = false;
+                    }
                     sb.append(token);
-                    if (token_iter.hasNext()) {
-                        sb.append(" ");
-                    }
                 }
             }
             else {
-                while (token_iter.hasNext()) {
-                    String token = token_iter.next();
+                boolean first_append = true;
+                
+                for (int i=0; i<tokens.size(); i++) {
+                    String token = tokens.get(i);
+                    
                     if (whitelist_bloomfilter.contains(token)) {
-                        sb.append(token);
-                        if (token_iter.hasNext()) {
+                        if (!first_append) {
                             sb.append(" ");
                         }
-                    }
-                }
-                
+                        else {
+                            first_append = false;
+                        }
+                        sb.append(token);
+                    }                   
+                }
+                    
             }
             /*
@@ -50 +107 @@
             }
     */
+            
+            
             
            return sb.toString();