Changeset 31220 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

Timestamp:

2016-12-12T20:18:04+13:00 (7 years ago)

Author:

davidb

Message:

Use of whitelist Bloom filter added to words going into Solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

@@ -25 +25 @@
     
     protected String _input_dir;
+    
+    protected WhitelistBloomFilter _whitelist_bloomfilter;
+
     protected String _solr_url;
     protected String _output_dir;
@@ -34 +37 @@
     boolean _strict_file_io;
     
-    public PerPageJSONFlatmap(String input_dir, String solr_url, String output_dir, int verbosity, 
+    public PerPageJSONFlatmap(String input_dir, String whitelist_filename, 
+                              String solr_url, String output_dir, int verbosity, 
                               DoubleAccumulator progress_accum, double progress_step,
                               boolean strict_file_io)
     {
         _input_dir  = input_dir;
+        
+        if (whitelist_filename != null) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+        }
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;
@@ -51 +60 @@
     
     public Iterator<JSONObject> call(String json_file_in) throws IOException
-    //public void call(String json_file_in) 
     { 
-            //ClusterFileIO.memory_usage("Before BZIP2 JSON file read");
-        String full_json_file_in = _input_dir + "/" + json_file_in;
+        String full_json_file_in = _input_dir + "/" + json_file_in;
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
-        //ClusterFileIO.memory_usage("After BZIP2 JSON file read");
-
+    
         ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>();