Changeset 31258 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk

Timestamp:

2016-12-20T23:39:40+13:00 (7 years ago)

Author:

davidb

Message:

POS Label count, similar to Whitelist word count

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Files:

• scripts/FULL-RUN-MASTER-SPARK-POS-COUNT.sh (added)
• src/main/java/org/hathitrust/extractedfeatures/PerVolumePOSStreamFlatmap.java (added)
• src/main/java/org/hathitrust/extractedfeatures/ProcessForPOSCount.java (added)
• src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -31 +31 @@
         if (ef_token_pos_count != null) {
 
-            Iterator<String> token_iter = ef_token_pos_count.keys();
-            while (token_iter.hasNext()) {
-                String token = token_iter.next();
+            Iterator<String> word_token_iter = ef_token_pos_count.keys();
+            while (word_token_iter.hasNext()) {
+                String word_token = word_token_iter.next();
                 
                 if (icu_tokenize == true) {
-                    Reader reader = new StringReader(token);
+                    Reader reader = new StringReader(word_token);
                     
                     ICUTokenizer icu_tokenizer = new ICUTokenizer();
@@ -68 +68 @@
                 }
                 else {
-                    words.add(token);
+                    words.add(word_token);
                 }
             }               
@@ -84 +84 @@
         return words;
     }
+    
+    protected static ArrayList<String> getTokenPosCountPOSLabels(JSONObject ef_token_pos_count, String page_id)
+    {
+        ArrayList<String> pos_labels = new ArrayList<String>();
+
+        if (ef_token_pos_count != null) {
+
+            Iterator<String> word_token_iter = ef_token_pos_count.keys();
+            while (word_token_iter.hasNext()) {
+                String word_token = word_token_iter.next();
+                
+                JSONObject word_pos_labels = ef_token_pos_count.getJSONObject(word_token);
+                
+                Iterator<String> pos_token_iter = word_pos_labels.keys();
+                while (pos_token_iter.hasNext()) {
+                    String pos_token = pos_token_iter.next();
+                        
+                    pos_labels.add(pos_token);
+                }
+            }               
+        }
+        else {
+            System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+        }
+
+        return pos_labels;
+    }
+
     
     
@@ -221 +249 @@
     }
 
-    protected static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page,
-                                                                 boolean icu_tokenize) 
+    public static ArrayList<String> generateTokenPosCountText(String volume_id, String page_id, JSONObject ef_page,
+                                                              boolean icu_tokenize) 
     {
         ArrayList<String> word_list = null;
@@ -241 +269 @@
         }
         
+        return word_list;
+    }
+    
+    public static ArrayList<String> generateTokenPosCountPOSLabels(String volume_id, String page_id, JSONObject ef_page) 
+    {
+        ArrayList<String> word_list = null;
+
+        if (ef_page != null) {
+            JSONObject ef_body = ef_page.getJSONObject("body");
+            if (ef_body != null) {
+                JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
+                word_list = getTokenPosCountPOSLabels(ef_token_pos_count,page_id);
+            }
+            else {
+                System.err.println("Warning: empty body field for '" + page_id + "'");
+            }
+
+        }
+        else {
+            System.err.println("Warning: null page for '" + page_id + "'");
+        }
+
         return word_list;
     }