Changeset 31220 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2016-12-12T20:18:04+13:00 (7 years ago)

Author:

davidb

Message:

Use of whitelist Bloom filter added to words going into Solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (5 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -33 +33 @@
     protected String _input_dir;
     protected String _json_list_filename;
+    protected String _whitelist_filename;
     protected String _solr_url;
     protected String _output_dir;
@@ -44 +45 @@
         _json_list_filename = (json_list_filename != null) ? json_list_filename : input_dir;
 
+        boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
+        _whitelist_filename = (use_whitelist) ?  System.getProperty("wcsa-ef-ingest.whitelist-filename") : null;
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;
@@ -117 +121 @@
         System.err.println();
         
-        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
+                                                       _solr_url,_output_dir,_verbosity, progress_accum,per_vol);
 
         json_list_data.foreach(per_vol_json);
@@ -160 +165 @@
                 
         PerPageJSONFlatmap paged_solr_json_flatmap 
-            = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, 
+            = new PerPageJSONFlatmap(_input_dir,_whitelist_filename,
+                                     _solr_url,_output_dir,_verbosity, 
                                      per_vol_progress_accum,per_vol,
                                      strict_file_io);
@@ -309 +315 @@
         ProcessForSolrIngest prep_for_ingest 
             = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
-        
-        
-        boolean use_whitelist = Boolean.getBoolean("wcsa-ef-ingest.use-whitelist");
-                
-        if (use_whitelist) {
-            String whitelist_filename = System.getProperty("wcsa-ef-ingest.whitelist-filename");
-
-            WhitelistBloomFilter whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
-            whitelist_bloomfilter.contains("foo");
-        }
-        
-        
+            
         String process_ef_json_mode = System.getProperty("wcsa-ef-ingest.process-ef-json-mode","per-page");
         if (process_ef_json_mode.equals("per-volume")) {