Changeset 31220 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

Timestamp:

2016-12-12T20:18:04+13:00 (7 years ago)

Author:

davidb

Message:

Use of whitelist Bloom filter added to words going into Solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -20 +20 @@
     private static final long serialVersionUID = 1L;
     protected String _input_dir;
+
+    protected WhitelistBloomFilter _whitelist_bloomfilter;
+
     protected String _solr_url;
     protected String _output_dir;
@@ -27 +30 @@
     protected double            _progress_step;
     
-    public PerVolumeJSON(String input_dir, String solr_url, String output_dir, int verbosity, 
+    public PerVolumeJSON(String input_dir, String whitelist_filename,
+                         String solr_url, String output_dir, int verbosity, 
                          DoubleAccumulator progress_accum, double progress_step)
     {
         _input_dir  = input_dir;
+        
+        if (whitelist_filename != null) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
+        }
+        
         _solr_url   = solr_url;
         _output_dir = output_dir;