Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/pom.xml
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/pom.xml	(revision 31242)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/pom.xml	(revision 31243)
@@ -51,5 +51,11 @@
       <version>20.0</version>
     </dependency>
-    
+
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analyzers-icu</artifactId>
+      <version>6.3.0</version>
+    </dependency>
+
   </dependencies>
 
@@ -64,4 +70,5 @@
 	   <source>1.8</source>
 	   <target>1.8</target>
+
 	 </configuration>
        </plugin>
Index: /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java
===================================================================
--- /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31242)
+++ /other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java	(revision 31243)
@@ -6,11 +6,18 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Set;
 
 import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONObject;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 
 public class SolrDocJSON {
@@ -18,29 +25,79 @@
 	protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
 		{
-			StringBuilder sb = new StringBuilder();
-	
+			boolean solr_icu_tokenize = true;
+		
+		    ArrayList<String> tokens = new ArrayList<String>();
+		    
 			Iterator<String> token_iter = ef_token_pos_count.keys();
 		    
+			while (token_iter.hasNext()) {
+				String token = token_iter.next();
+
+				if (solr_icu_tokenize == true) {
+					Reader reader = new StringReader(token);
+					
+					Tokenizer tokenizer = new ICUTokenizer();
+					tokenizer.setReader(reader);
+						
+					//TokenStream tokenStream = analyzer.tokenStream(fieldName, reader);
+					//OffsetAttribute offsetAttribute = tokenizer.addAttribute(OffsetAttribute.class);
+					CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
+
+					try {
+						tokenizer.reset();
+						
+						while (tokenizer.incrementToken()) {
+						    //int startOffset = offsetAttribute.startOffset();
+						    //int endOffset = offsetAttribute.endOffset();
+						    String term = charTermAttribute.toString();
+						    tokens.add(term);
+						}
+						
+						tokenizer.close();
+					} 
+					catch (IOException e) {
+						e.printStackTrace();
+					}
+				}
+				else {
+					tokens.add(token);
+				}
+			}
+			
+			StringBuilder sb = new StringBuilder();
+			
 			if (whitelist_bloomfilter == null) {
 
-				while (token_iter.hasNext()) {
-					String token = token_iter.next();
+				boolean first_append = true;
+				
+				for (int i=0; i<tokens.size(); i++) {
+					String token = tokens.get(i);
+									
+					if (!first_append) {
+						sb.append(" ");
+					}
+					else {
+						first_append = false;
+					}
 					sb.append(token);
-					if (token_iter.hasNext()) {
-						sb.append(" ");
-					}
 				}
 			}
 			else {
-				while (token_iter.hasNext()) {
-					String token = token_iter.next();
+				boolean first_append = true;
+				
+				for (int i=0; i<tokens.size(); i++) {
+					String token = tokens.get(i);
+					
 					if (whitelist_bloomfilter.contains(token)) {
-						sb.append(token);
-						if (token_iter.hasNext()) {
+						if (!first_append) {
 							sb.append(" ");
 						}
-					}
-				}
-				
+						else {
+							first_append = false;
+						}
+						sb.append(token);
+					}					
+				}
+					
 			}
 			/*
@@ -50,4 +107,6 @@
 			}
 	*/
+			
+			
 			
 		   return sb.toString();
