Changeset 31176 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-03T21:16:38+13:00 (7 years ago)

Author:

davidb

Message:

Support added for producing whitelist word count

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -8 +8 @@
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Iterator;
 
@@ -123 +124 @@
     }
 
+    protected static ArrayList<String> generateTokenPostCountText(String volume_id, String page_id, JSONObject ef_page) 
+    {
+        ArrayList<String> word_list = new ArrayList<String>();
+        
+        if (ef_page != null) {
+            JSONObject ef_body = ef_page.getJSONObject("body");
+            if (ef_body != null) {
+                JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
+                if (ef_token_pos_count != null) {
+    
+                    Iterator<String> token_iter = ef_token_pos_count.keys();
+                    while (token_iter.hasNext()) {
+                        String token = token_iter.next();
+                        word_list.add(token);
+                    }               
+                }
+                else {
+                    System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+                }
+            }
+            else {
+                System.err.println("Warning: empty body field for '" + page_id + "'");
+            }
+
+        }
+        else {
+            System.err.println("Warning: null page for '" + page_id + "'");
+        }
+        
+        return word_list;
+    }
+    
     public static void saveSolrDoc(JSONObject solr_add_doc_json, String output_file_json_bz2)
     {