Ignore:
Timestamp:
10/26/16 14:27:44 (4 years ago)
Author:
davidb
Message:

Improved output printing for slave node

Location:
other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

    r30937 r30942  
    2424
    2525    protected String _input_dir;
     26    protected int    _verbosity;
    2627   
    27     public PagedJSON(String input_dir)
     28    public PagedJSON(String input_dir, int verbosity)
    2829    {
    2930        _input_dir = input_dir;
     31        _verbosity = verbosity;
    3032    }
    3133   
     
    8789        int ef_page_count = ef_features.getInt("pageCount");
    8890       
     91        if (_verbosity >= 1) {
     92            System.out.println("Processing: " + json_file_in);
     93            System.out.println("  pageCount = " + ef_page_count);
     94        }
     95       
    8996        JSONArray ef_pages = ef_features.getJSONArray("pages");
    9097        int ef_num_pages = ef_pages.length();
  • other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

    r30941 r30942  
    4747        JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename).cache();
    4848
    49         JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir));
     49        JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir,_verbosity));
    5050
    5151
     
    6363        System.out.println("");
    6464        System.out.println("############");
    65         System.out.println("# number of IDS: " + num_ids);
     65        System.out.println("# Number of page ids: " + num_ids);
    6666        System.out.println("############");
    6767        System.out.println("");
Note: See TracChangeset for help on using the changeset viewer.