Changeset 31220 for other-projects/hathitrust/wcsa

Timestamp:

2016-12-12T20:18:04+13:00 (7 years ago)

Author:

davidb

Message:

Use of whitelist Bloom filter added to words going into Solr index

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerPageJSONFlatmap.java (modified) (3 diffs)
• PerVolumeJSON.java (modified) (2 diffs)
• ProcessForSolrIngest.java (modified) (5 diffs)
• SolrDocJSON.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

@@ -25 +25 @@
     
     protected String _input_dir;
+    
+    protected WhitelistBloomFilter _whitelist_bloomfilter;
+
     protected String _solr_url;
     protected String _output_dir;
@@ -34 +37 @@
     boolean _strict_file_io;
     
-    public PerPageJSONFlatmap(String input_dir, String solr_url, String output_dir, int verbosity, 
+    public PerPageJSONFlatmap(String input_dir, String whitelist_filename, 
+                              String solr_url, String output_dir, int verbosity, 
                               DoubleAccumulator progress_accum, double progress_step,
                               boolean strict_file_io)
     {
         _input_dir  = input_dir;
+        
+        if (whitelist_filename != null) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+        }
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;
@@ -51 +60 @@
     
     public Iterator<JSONObject> call(String json_file_in) throws IOException
-    //public void call(String json_file_in) 
     { 
-            //ClusterFileIO.memory_usage("Before BZIP2 JSON file read");
-        String full_json_file_in = _input_dir + "/" + json_file_in;
+        String full_json_file_in = _input_dir + "/" + json_file_in;
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
-        //ClusterFileIO.memory_usage("After BZIP2 JSON file read");
-
+    
         ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>();
         

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -20 +20 @@
     private static final long serialVersionUID = 1L;
     protected String _input_dir;
+
+    protected WhitelistBloomFilter _whitelist_bloomfilter;
+
     protected String _solr_url;
     protected String _output_dir;
@@ -27 +30 @@
     protected double            _progress_step;
     
-    public PerVolumeJSON(String input_dir, String solr_url, String output_dir, int verbosity, 
+    public PerVolumeJSON(String input_dir, String whitelist_filename,
+                         String solr_url, String output_dir, int verbosity, 
                          DoubleAccumulator progress_accum, double progress_step)
     {
         _input_dir  = input_dir;
+        
+        if (whitelist_filename != null) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+        }
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -33 +33 @@
     protected String _input_dir;
     protected String _json_list_filename;
+    protected String _whitelist_filename;
     protected String _solr_url;
     protected String _output_dir;
@@ -44 +45 @@
         _json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
 
+        boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
+        _whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;
@@ -117 +121 @@
         System.err.println();
         
-        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+                                                       _solr_url,_output_dir,_verbosity, progress_accum,per_vol);
 
         json_list_data.foreach(per_vol_json);
@@ -160 +165 @@
                 
         PerPageJSONFlatmap paged_solr_json_flatmap 
-            = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, 
+            = new PerPageJSONFlatmap(_input_dir,_whitelist_filename,
+                                     _solr_url,_output_dir,_verbosity, 
                                      per_vol_progress_accum,per_vol,
                                      strict_file_io);
@@ -309 +315 @@
         ProcessForSolrIngest prep_for_ingest 
             = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
-        
-        
-        boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
-                
-        if (use_whitelist) {
-            String whitelist_filename = System.getProperty("wcsa-ef-ingest.whitelist-filename");
-
-            WhitelistBloomFilter whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
-            whitelist_bloomfilter.contains("foo");
-        }
-        
-        
+            
         String process_ef_json_mode = System.getProperty("wcsa-ef-ingest.process-ef-json-mode","per-page");
         if (process_ef_json_mode.equals("per-volume")) { 

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -16 +16 @@
 public class SolrDocJSON {
 
-    protected static String generateSolrText(JSONObject ef_token_pos_count)
+    protected static String generateSolrText(JSONObject ef_token_pos_count, WhitelistBloomFilter whitelist_bloomfilter)
         {
             StringBuilder sb = new StringBuilder();
     
             Iterator<String> token_iter = ef_token_pos_count.keys();
-            while (token_iter.hasNext()) {
-                String token = token_iter.next();
-                
-                sb.append(token);
-                if (token_iter.hasNext()) {
-                    sb.append(" ");
-                }
-            }
-            
+            
+            if (whitelist_bloomfilter == null) {
+
+                while (token_iter.hasNext()) {
+                    String token = token_iter.next();
+                    sb.append(token);
+                    if (token_iter.hasNext()) {
+                        sb.append(" ");
+                    }
+                }
+            }
+            else {
+                while (token_iter.hasNext()) {
+                    String token = token_iter.next();
+                    if (whitelist_bloomfilter.contains(token)) {
+                        sb.append(token);
+                        if (token_iter.hasNext()) {
+                            sb.append(" ");
+                        }
+                    }
+                }
+                
+            }
             /*
             Set<String> token_keys = ef_token_pos_count.keySet();
@@ -40 +54 @@
         }
 
-    protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page) 
+    protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
+                                                    WhitelistBloomFilter whitelist_bloomfilter) 
     {
         JSONObject solr_update_json = null;
@@ -52 +67 @@
                     JSONObject solr_add_json = new JSONObject();
     
-                    String text = generateSolrText(ef_token_pos_count);
+                    String text = generateSolrText(ef_token_pos_count,whitelist_bloomfilter);
                     
                     JSONObject solr_doc_json = new JSONObject();
                     solr_doc_json.put("id", page_id);
                     solr_doc_json.put("volumeid_s", volume_id);
-                    solr_doc_json.put("eftext_txt", text);
-                   
+                    if (!text.equals("")) {
+                        solr_doc_json.put("eftext_txt", text);
+                    }
+                    else {
+                        solr_doc_json.put("efnotext_b", true);
+                    }
                     solr_add_json.put("commitWithin", 5000);
                     solr_add_json.put("doc", solr_doc_json);