Changeset 31245 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-18T17:18:13+13:00 (7 years ago)

Author:

davidb

Message:

Refactored so processing of words from TokenPosCount now done by the same sub-routine

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (7 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -24 +24 @@
 public class SolrDocJSON {
 
-    protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
-        {
-            boolean solr_icu_tokenize = true;
-        
-            ArrayList<String> tokens = new ArrayList<String>();
-            
+    protected static ArrayList<String> getTokenPosCountWords(JSONObject ef_token_pos_count, String page_id)
+    {
+        boolean solr_icu_tokenize = true;
+        
+        ArrayList<String> words = new ArrayList<String>();
+        
+        if (ef_token_pos_count != null) {
+
             Iterator<String> token_iter = ef_token_pos_count.keys();
-            
             while (token_iter.hasNext()) {
                 String token = token_iter.next();
-
+                
                 if (solr_icu_tokenize == true) {
                     Reader reader = new StringReader(token);
@@ -48 +49 @@
                         while (tokenizer.incrementToken()) {
                             String term = charTermAttribute.toString();
-                            tokens.add(term);
+                            words.add(term);
                         }
                         
@@ -59 +60 @@
                 }
                 else {
-                    tokens.add(token);
-                }
-            }
-            
-            StringBuilder sb = new StringBuilder();
-            
-            if (whitelist_bloomfilter == null) {
-
-                boolean first_append = true;
-                
-                for (int i=0; i<tokens.size(); i++) {
-                    String token = tokens.get(i);
-                                    
+                    words.add(token);
+                }
+            }               
+        }
+        else {
+            System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+        }
+        
+        /* Alternative way to get at keys
+        Set<String> token_keys = ef_token_pos_count.keySet();
+        for (String token : token_keys) {
+                sb.append(token + " ");
+        }
+*/
+        return words;
+    }
+    
+    
+    protected static String generateSolrText(JSONObject ef_token_pos_count, String page_id,
+                                            WhitelistBloomFilter whitelist_bloomfilter)
+    {
+        ArrayList<String> tokens = getTokenPosCountWords(ef_token_pos_count, page_id);
+
+        StringBuilder sb = new StringBuilder();
+
+        if (whitelist_bloomfilter == null) {
+
+            boolean first_append = true;
+
+            for (int i=0; i<tokens.size(); i++) {
+                String token = tokens.get(i);
+
+                if (!first_append) {
+                    sb.append(" ");
+                }
+                else {
+                    first_append = false;
+                }
+                sb.append(token);
+            }
+        }
+        else {
+            boolean first_append = true;
+
+            for (int i=0; i<tokens.size(); i++) {
+                String token = tokens.get(i);
+
+                if (whitelist_bloomfilter.contains(token)) {
                     if (!first_append) {
                         sb.append(" ");
@@ -79 +115 @@
                     }
                     sb.append(token);
-                }
-            }
-            else {
-                boolean first_append = true;
-                
-                for (int i=0; i<tokens.size(); i++) {
-                    String token = tokens.get(i);
-                    
-                    if (whitelist_bloomfilter.contains(token)) {
-                        if (!first_append) {
-                            sb.append(" ");
-                        }
-                        else {
-                            first_append = false;
-                        }
-                        sb.append(token);
-                    }                   
-                }
-                    
-            }
-            /*
-            Set<String> token_keys = ef_token_pos_count.keySet();
-            for (String token : token_keys) {
-                    sb.append(token + " ");
-            }
-    */
-            
-           return sb.toString();
-        }
+                }                   
+            }
+
+        }
+
+
+        return sb.toString();
+    }
 
     protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
@@ -122 +137 @@
                     JSONObject solr_add_json = new JSONObject();
     
-                    String text = generateSolrText(ef_token_pos_count,whitelist_bloomfilter);
+                    String text = generateSolrText(ef_token_pos_count,page_id,whitelist_bloomfilter);
                     
                     JSONObject solr_doc_json = new JSONObject();
@@ -200 +215 @@
     protected static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page) 
     {
-        ArrayList<String> word_list = new ArrayList<String>();
+        ArrayList<String> word_list = null;
         
         if (ef_page != null) {
@@ -206 +221 @@
             if (ef_body != null) {
                 JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
-                if (ef_token_pos_count != null) {
-    
-                    Iterator<String> token_iter = ef_token_pos_count.keys();
-                    while (token_iter.hasNext()) {
-                        String token = token_iter.next();
-                        word_list.add(token);
-                    }               
-                }
-                else {
-                    System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
-                }
+                word_list = getTokenPosCountWords(ef_token_pos_count,page_id);
             }
             else {