Changeset 30942 for other-projects

Show
Ignore:
Timestamp:
26.10.2016 14:27:44 (3 years ago)
Author:
davidb
Message:

Improved output printing for slave node

Location:
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust
Files:
2 modified

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

    r30937 r30942  
    2424 
    2525    protected String _input_dir; 
     26    protected int    _verbosity; 
    2627     
    27     public PagedJSON(String input_dir) 
     28    public PagedJSON(String input_dir, int verbosity) 
    2829    { 
    2930        _input_dir = input_dir; 
     31        _verbosity = verbosity; 
    3032    } 
    3133     
     
    8789        int ef_page_count = ef_features.getInt("pageCount"); 
    8890         
     91        if (_verbosity >= 1) { 
     92            System.out.println("Processing: " + json_file_in); 
     93            System.out.println("  pageCount = " + ef_page_count); 
     94        } 
     95         
    8996        JSONArray ef_pages = ef_features.getJSONArray("pages"); 
    9097        int ef_num_pages = ef_pages.length(); 
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

    r30941 r30942  
    4747        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename).cache(); 
    4848 
    49         JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir)); 
     49        JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir,_verbosity)); 
    5050 
    5151 
     
    6363        System.out.println(""); 
    6464        System.out.println("############"); 
    65         System.out.println("# number of IDS: " + num_ids); 
     65        System.out.println("# Number of page ids: " + num_ids); 
    6666        System.out.println("############"); 
    6767        System.out.println("");