Changeset 31006


Ignore:
Timestamp:
2016-10-31T09:27:42+13:00 (6 years ago)
Author:
davidb
Message:

Further reversal of Base class. Switch to PerPage

Location:
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java

    r31003 r31006  
    44import java.util.Iterator;
    55import org.apache.spark.api.java.function.FlatMapFunction;
    6 import org.apache.spark.api.java.function.VoidFunction;
    76import org.apache.spark.util.DoubleAccumulator;
    87import org.json.JSONArray;
     
    1918
    2019
    21 class PagedJSON extends BasePerJSON implements FlatMapFunction<String, JSONObject>
     20class PagedJSON implements FlatMapFunction<String, JSONObject>
    2221//public class PagedJSON implements VoidFunction<String>
    2322{
    2423    private static final long serialVersionUID = 1L;
    2524   
    26     public PagedJSON()
    27     {
    28         super();
    29     }
     25    protected String _input_dir;
     26    protected String _solr_url;
     27    protected String _output_dir;
     28    protected int    _verbosity;
     29   
     30    protected DoubleAccumulator _progress_accum;
     31    protected double            _progress_step;
     32   
    3033    public PagedJSON(String input_dir, String solr_url, String output_dir, int verbosity,
    3134                     DoubleAccumulator progress_accum, double progress_step)
    3235    {
    33         super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step);
     36        _input_dir  = input_dir;
     37        _solr_url   = solr_url;
     38        _output_dir = output_dir;
     39        _verbosity  = verbosity;
     40       
     41        _progress_accum = progress_accum;
     42        _progress_step  = progress_step;
    3443    }
    35    
    3644   
    3745    public Iterator<JSONObject> call(String json_file_in)
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSONForeach.java

    r31003 r31006  
    1919
    2020
    21 class PagedJSONForeach extends BasePerJSON implements VoidFunction<JSONObject>
     21class PagedJSONForeach implements VoidFunction<JSONObject>
    2222//public class PagedJSON implements VoidFunction<String>
    2323{
    2424    private static final long serialVersionUID = 1L;
    2525   
    26     public PagedJSONForeach()
    27     {
    28         super();
    29     }
     26    protected String _input_dir;
     27    protected String _solr_url;
     28    protected String _output_dir;
     29    protected int    _verbosity;
     30   
     31    protected DoubleAccumulator _progress_accum;
     32    protected double            _progress_step;
     33   
    3034   
    3135    public PagedJSONForeach(String input_dir, String solr_url, String output_dir, int verbosity,
    32                             DoubleAccumulator progress_accum, double progress_step)
     36                            DoubleAccumulator progress_accum, double progress_step)
    3337    {
    34         super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step);
     38        _input_dir  = input_dir;
     39        _solr_url   = solr_url;
     40        _output_dir = output_dir;
     41        _verbosity  = verbosity;
     42       
     43        _progress_accum = progress_accum;
     44        _progress_step  = progress_step;
    3545    }
    36    
    3746   
    3847    public void call(JSONObject solr_add_doc_json)
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

    r31004 r31006  
    220220        ProcessForSolrIngest prep_for_ingest
    221221            = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity);
    222         prep_for_ingest.execPerVolume();
     222       
     223        //prep_for_ingest.execPerVolume();
     224        prep_for_ingest.execPerPage();
    223225    }
    224226}
Note: See TracChangeset for help on using the changeset viewer.