Changeset 31220 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

Timestamp:

2016-12-12T20:18:04+13:00 (7 years ago)

Author:

davidb

Message:

Use of whitelist Bloom filter added to words going into Solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -16 +16 @@
 public class SolrDocJSON {
 
-    protected static String generateSolrText(JSONObject ef_token_pos_count)
+    protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
         {
             StringBuilder sb = new StringBuilder();
     
             Iterator<String> token_iter = ef_token_pos_count.keys();
-            while (token_iter.hasNext()) {
-                String token = token_iter.next();
-                
-                sb.append(token);
-                if (token_iter.hasNext()) {
-                    sb.append(" ");
-                }
-            }
-            
+            
+            if (whitelist_bloomfilter == null) {
+
+                while (token_iter.hasNext()) {
+                    String token = token_iter.next();
+                    sb.append(token);
+                    if (token_iter.hasNext()) {
+                        sb.append(" ");
+                    }
+                }
+            }
+            else {
+                while (token_iter.hasNext()) {
+                    String token = token_iter.next();
+                    if (whitelist_bloomfilter.contains(token)) {
+                        sb.append(token);
+                        if (token_iter.hasNext()) {
+                            sb.append(" ");
+                        }
+                    }
+                }
+                
+            }
             /*
             Set<String> token_keys = ef_token_pos_count.keySet();
@@ -40 +54 @@
         }
 
-    protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page) 
+    protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
+                                                    WhitelistBloomFilter whitelist_bloomfilter) 
     {
         JSONObject solr_update_json = null;
@@ -52 +67 @@
                     JSONObject solr_add_json = new JSONObject();
     
-                    String text = generateSolrText(ef_token_pos_count);
+                    String text = generateSolrText(ef_token_pos_count,whitelist_bloomfilter);
                     
                     JSONObject solr_doc_json = new JSONObject();
                     solr_doc_json.put("id", page_id);
                     solr_doc_json.put("volumeid_s", volume_id);
-                    solr_doc_json.put("eftext_txt", text);
-                   
+                    if (!text.equals("")) {
+                        solr_doc_json.put("eftext_txt", text);
+                    }
+                    else {
+                        solr_doc_json.put("efnotext_b", true);
+                    }
                     solr_add_json.put("commitWithin", 5000);
                     solr_add_json.put("doc", solr_doc_json);