Changeset 31006

Show
Ignore:
Timestamp:
31.10.2016 09:27:42 (3 years ago)
Author:
davidb
Message:

Further reversal of Base class. Switch to PerPage?

Location:
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures
Files:
3 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java

    r31003 r31006  
    44import java.util.Iterator; 
    55import org.apache.spark.api.java.function.FlatMapFunction; 
    6 import org.apache.spark.api.java.function.VoidFunction; 
    76import org.apache.spark.util.DoubleAccumulator; 
    87import org.json.JSONArray; 
     
    1918 
    2019 
    21 class PagedJSON extends BasePerJSON implements FlatMapFunction<String, JSONObject> 
     20class PagedJSON implements FlatMapFunction<String, JSONObject> 
    2221//public class PagedJSON implements VoidFunction<String>  
    2322{ 
    2423    private static final long serialVersionUID = 1L; 
    2524     
    26     public PagedJSON() 
    27     { 
    28         super(); 
    29     } 
     25    protected String _input_dir; 
     26    protected String _solr_url; 
     27    protected String _output_dir; 
     28    protected int    _verbosity; 
     29     
     30    protected DoubleAccumulator _progress_accum; 
     31    protected double            _progress_step; 
     32     
    3033    public PagedJSON(String input_dir, String solr_url, String output_dir, int verbosity,  
    3134                     DoubleAccumulator progress_accum, double progress_step) 
    3235    { 
    33         super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step); 
     36        _input_dir  = input_dir; 
     37        _solr_url   = solr_url; 
     38        _output_dir = output_dir; 
     39        _verbosity  = verbosity; 
     40         
     41        _progress_accum = progress_accum; 
     42        _progress_step  = progress_step; 
    3443    } 
    35      
    3644     
    3745    public Iterator<JSONObject> call(String json_file_in)  
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSONForeach.java

    r31003 r31006  
    1919 
    2020 
    21 class PagedJSONForeach extends BasePerJSON implements VoidFunction<JSONObject>  
     21class PagedJSONForeach implements VoidFunction<JSONObject>  
    2222//public class PagedJSON implements VoidFunction<String>  
    2323{ 
    2424    private static final long serialVersionUID = 1L; 
    2525     
    26     public PagedJSONForeach() 
    27     { 
    28         super(); 
    29     } 
     26    protected String _input_dir; 
     27    protected String _solr_url; 
     28    protected String _output_dir; 
     29    protected int    _verbosity; 
     30     
     31    protected DoubleAccumulator _progress_accum; 
     32    protected double            _progress_step; 
     33     
    3034     
    3135    public PagedJSONForeach(String input_dir, String solr_url, String output_dir, int verbosity,  
    32                             DoubleAccumulator progress_accum, double progress_step) 
     36                            DoubleAccumulator progress_accum, double progress_step) 
    3337    { 
    34         super(input_dir,solr_url,output_dir,verbosity,progress_accum,progress_step); 
     38        _input_dir  = input_dir; 
     39        _solr_url   = solr_url; 
     40        _output_dir = output_dir; 
     41        _verbosity  = verbosity; 
     42         
     43        _progress_accum = progress_accum; 
     44        _progress_step  = progress_step; 
    3545    } 
    36      
    3746     
    3847    public void call(JSONObject solr_add_doc_json)  
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

    r31004 r31006  
    220220        ProcessForSolrIngest prep_for_ingest  
    221221            = new ProcessForSolrIngest(input_dir,json_list_filename,solr_url,output_dir,verbosity); 
    222         prep_for_ingest.execPerVolume(); 
     222         
     223        //prep_for_ingest.execPerVolume(); 
     224        prep_for_ingest.execPerPage(); 
    223225    } 
    224226}