Changeset 31006 for other-projects/hathitrust
- Timestamp:
- 2016-10-31T09:27:42+13:00 (7 years ago)
- Location:
- other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java
r31003 r31006 4 4 import java.util.Iterator; 5 5 import org.apache.spark.api.java.function.FlatMapFunction; 6 import org.apache.spark.api.java.function.VoidFunction;7 6 import org.apache.spark.util.DoubleAccumulator; 8 7 import org.json.JSONArray; … … 19 18 20 19 21 class PagedJSON extends BasePerJSONimplements FlatMapFunction<String, JSONObject>20 class PagedJSON implements FlatMapFunction<String, JSONObject> 22 21 //public class PagedJSON implements VoidFunction<String> 23 22 { 24 23 private static final long serialVersionUID = 1L; 25 24 26 public PagedJSON() 27 { 28 super(); 29 } 25 protected String _input_dir; 26 protected String _solr_url; 27 protected String _output_dir; 28 protected int _verbosity; 29 30 protected DoubleAccumulator _progress_accum; 31 protected double _progress_step; 32 30 33 public PagedJSON(String input_dir, String solr_url, String output_dir, int verbosity, 31 34 DoubleAccumulator progress_accum, double progress_step) 32 35 { 33 super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step); 36 _input_dir = input_dir; 37 _solr_url = solr_url; 38 _output_dir = output_dir; 39 _verbosity = verbosity; 40 41 _progress_accum = progress_accum; 42 _progress_step = progress_step; 34 43 } 35 36 44 37 45 public Iterator<JSONObject> call(String json_file_in) -
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSONForeach.java
r31003 r31006 19 19 20 20 21 class PagedJSONForeach extends BasePerJSONimplements VoidFunction<JSONObject>21 class PagedJSONForeach implements VoidFunction<JSONObject> 22 22 //public class PagedJSON implements VoidFunction<String> 23 23 { 24 24 private static final long serialVersionUID = 1L; 25 25 26 public PagedJSONForeach() 27 { 28 super(); 29 } 26 protected String _input_dir; 27 protected String _solr_url; 28 protected String _output_dir; 29 protected int _verbosity; 30 31 protected DoubleAccumulator _progress_accum; 32 protected double _progress_step; 33 30 34 31 35 public PagedJSONForeach(String input_dir, String solr_url, String output_dir, int verbosity, 32 36 DoubleAccumulator progress_accum, double progress_step) 33 37 { 34 super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step); 38 _input_dir = input_dir; 39 _solr_url = solr_url; 40 _output_dir = output_dir; 41 _verbosity = verbosity; 42 43 _progress_accum = progress_accum; 44 _progress_step = progress_step; 35 45 } 36 37 46 38 47 public void call(JSONObject solr_add_doc_json) -
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java
r31004 r31006 220 220 ProcessForSolrIngest prep_for_ingest 221 221 = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity); 222 prep_for_ingest.execPerVolume(); 222 223 //prep_for_ingest.execPerVolume(); 224 prep_for_ingest.execPerPage(); 223 225 } 224 226 }
Note:
See TracChangeset
for help on using the changeset viewer.