Changeset 31225 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest

Timestamp:

2016-12-13T13:46:23+13:00 (7 years ago)

Author:

davidb

Message:

Relocated bloomfilter creation to within call() method, so done on the submitted side where the code runs

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerPageJSONFlatmap.java (modified) (4 diffs)
• PerVolumeJSON.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

@@ -25 +25 @@
     
     protected String _input_dir;
+    protected String _whitelist_filename;
+
+    protected String _solr_url;
+    protected String _output_dir;
+    
+    protected int    _verbosity;
     
     protected WhitelistBloomFilter _whitelist_bloomfilter;
 
-    protected String _solr_url;
-    protected String _output_dir;
-    protected int    _verbosity;
     
     protected DoubleAccumulator _progress_accum;
@@ -43 +46 @@
     {
         _input_dir  = input_dir;
-        
-        if (whitelist_filename != null) {
-            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
-        }
+        _whitelist_filename = whitelist_filename;
         
         _solr_url   = solr_url;
@@ -56 +56 @@
         
         _strict_file_io = strict_file_io;
+        
+        _whitelist_bloomfilter = null;
     }
     
@@ -61 +63 @@
     public Iterator<JSONObject> call(String json_file_in) throws IOException
     { 
+        if ((_whitelist_filename != null) && (_whitelist_bloomfilter != null)) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(_whitelist_filename,true);
+        }
+        
         String full_json_file_in = _input_dir + "/" + json_file_in;
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -20 +20 @@
     private static final long serialVersionUID = 1L;
     protected String _input_dir;
-
-    protected WhitelistBloomFilter _whitelist_bloomfilter;
-
+    protected String _whitelist_filename;
+    
     protected String _solr_url;
     protected String _output_dir;
+    
     protected int    _verbosity;
+    
+    protected WhitelistBloomFilter _whitelist_bloomfilter;
     
     protected DoubleAccumulator _progress_accum;
@@ -35 +37 @@
     {
         _input_dir  = input_dir;
-        
-        if (whitelist_filename != null) {
-            _whitelist_bloomfilter = new WhitelistBloomFilter(whitelist_filename,true);
-        }
+        _whitelist_filename = whitelist_filename;
         
         _solr_url   = solr_url;
@@ -46 +45 @@
         _progress_accum = progress_accum;
         _progress_step  = progress_step;
+        
+        _whitelist_bloomfilter = null;
     }
     
@@ -51 +52 @@
     public void call(String json_file_in) 
     { 
+        if ((_whitelist_filename != null) && (_whitelist_bloomfilter != null)) {
+            _whitelist_bloomfilter = new WhitelistBloomFilter(_whitelist_filename,true);
+        }
+
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(_input_dir + "/" + json_file_in);