Changeset 31252 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

Timestamp:

2016-12-20T14:15:05+13:00 (7 years ago)

Author:

davidb

Message:

Support for icu-tokenize property added, plus relevant refactoring.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (7 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -24 +24 @@
 public class SolrDocJSON {
 
-    protected static ArrayList<String> getTokenPosCountWords(JSONObject ef_token_pos_count, String page_id)
-    {
-        boolean solr_icu_tokenize = true;
+    protected static ArrayList<String> getTokenPosCountWords(JSONObject ef_token_pos_count, String page_id,
+                                                             boolean icu_tokenize)
+    {
         
         ArrayList<String> words = new ArrayList<String>();
@@ -36 +36 @@
                 String token = token_iter.next();
                 
-                if (solr_icu_tokenize == true) {
+                if (icu_tokenize == true) {
                     Reader reader = new StringReader(token);
                     
@@ -79 +79 @@
     
     protected static String generateSolrText(JSONObject ef_token_pos_count, String page_id,
-                                            WhitelistBloomFilter whitelist_bloomfilter)
-    {
-        ArrayList<String> tokens = getTokenPosCountWords(ef_token_pos_count, page_id);
+                                            WhitelistBloomFilter whitelist_bloomfilter, boolean icu_tokenize)
+    {
+        ArrayList<String> tokens = getTokenPosCountWords(ef_token_pos_count, page_id,icu_tokenize);
 
         StringBuilder sb = new StringBuilder();
@@ -125 +125 @@
 
     protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
-                                                    WhitelistBloomFilter whitelist_bloomfilter) 
+                                                    WhitelistBloomFilter whitelist_bloomfilter, boolean icu_tokenize) 
     {
         JSONObject solr_update_json = null;
@@ -137 +137 @@
                     JSONObject solr_add_json = new JSONObject();
     
-                    String text = generateSolrText(ef_token_pos_count,page_id,whitelist_bloomfilter);
+                    String text = generateSolrText(ef_token_pos_count,page_id,whitelist_bloomfilter,icu_tokenize);
                     
                     JSONObject solr_doc_json = new JSONObject();
@@ -213 +213 @@
     }
 
-    protected static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page) 
+    protected static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page,
+                                                                 boolean icu_tokenize) 
     {
         ArrayList<String> word_list = null;
@@ -221 +222 @@
             if (ef_body != null) {
                 JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
-                word_list = getTokenPosCountWords(ef_token_pos_count,page_id);
+                word_list = getTokenPosCountWords(ef_token_pos_count,page_id,icu_tokenize);
             }
             else {