Changeset 31006

Timestamp:

2016-10-31T09:27:42+13:00 (7 years ago)

Author:

davidb

Message:

Further reversal of Base class. Switch to PerPage

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures

Files:

• PagedJSON.java (modified) (2 diffs)
• PagedJSONForeach.java (modified) (1 diff)
• ProcessForSolrIngest.java (modified) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java

@@ -4 +4 @@
 import java.util.Iterator;
 import org.apache.spark.api.java.function.FlatMapFunction;
-import org.apache.spark.api.java.function.VoidFunction;
 import org.apache.spark.util.DoubleAccumulator;
 import org.json.JSONArray;
@@ -19 +18 @@
 
 
-class PagedJSON extends BasePerJSON implements FlatMapFunction<String, JSONObject>
+class PagedJSON implements FlatMapFunction<String, JSONObject>
 //public class PagedJSON implements VoidFunction<String> 
 {
     private static final long serialVersionUID = 1L;
     
-    public PagedJSON()
-    {
-        super();
-    }
+    protected String _input_dir;
+    protected String _solr_url;
+    protected String _output_dir;
+    protected int    _verbosity;
+    
+    protected DoubleAccumulator _progress_accum;
+    protected double            _progress_step;
+    
     public PagedJSON(String input_dir, String solr_url, String output_dir, int verbosity, 
                      DoubleAccumulator progress_accum, double progress_step)
     {
-        super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step);
+        _input_dir  = input_dir;
+        _solr_url   = solr_url;
+        _output_dir = output_dir;
+        _verbosity  = verbosity;
+        
+        _progress_accum = progress_accum;
+        _progress_step  = progress_step;
     }
-    
     
     public Iterator<JSONObject> call(String json_file_in) 

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSONForeach.java

@@ -19 +19 @@
 
 
-class PagedJSONForeach extends BasePerJSON implements VoidFunction<JSONObject> 
+class PagedJSONForeach implements VoidFunction<JSONObject> 
 //public class PagedJSON implements VoidFunction<String> 
 {
     private static final long serialVersionUID = 1L;
     
-    public PagedJSONForeach()
-    {
-        super();
-    }
+    protected String _input_dir;
+    protected String _solr_url;
+    protected String _output_dir;
+    protected int    _verbosity;
+    
+    protected DoubleAccumulator _progress_accum;
+    protected double            _progress_step;
+    
     
     public PagedJSONForeach(String input_dir, String solr_url, String output_dir, int verbosity, 
-                            DoubleAccumulator progress_accum, double progress_step)
+                            DoubleAccumulator progress_accum, double progress_step)
     {
-        super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step);
+        _input_dir  = input_dir;
+        _solr_url   = solr_url;
+        _output_dir = output_dir;
+        _verbosity  = verbosity;
+        
+        _progress_accum = progress_accum;
+        _progress_step  = progress_step;
     }
-    
     
     public void call(JSONObject solr_add_doc_json) 

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -220 +220 @@
         ProcessForSolrIngest prep_for_ingest 
             = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
-        prep_for_ingest.execPerVolume();
+        
+        //prep_for_ingest.execPerVolume();
+        prep_for_ingest.execPerPage();
     }
 }