Changeset 30942 for other-projects/hathitrust/solr-extracted-features

Timestamp:

2016-10-26T14:27:44+13:00 (7 years ago)

Author:

davidb

Message:

Improved output printing for slave node

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust

Files:

• PagedJSON.java (modified) (2 diffs)
• PrepareForIngest.java (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -24 +24 @@
 
     protected String _input_dir;
+    protected int    _verbosity;
     
-    public PagedJSON(String input_dir)
+    public PagedJSON(String input_dir, int verbosity)
     {
         _input_dir = input_dir;
+        _verbosity = verbosity;
     }
     
@@ -87 +89 @@
         int ef_page_count = ef_features.getInt("pageCount");
         
+        if (_verbosity >= 1) {
+            System.out.println("Processing: " + json_file_in);
+            System.out.println("  pageCount = " + ef_page_count);
+        }
+        
         JSONArray ef_pages = ef_features.getJSONArray("pages");
         int ef_num_pages = ef_pages.length();

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

@@ -47 +47 @@
         JavaRDD<String> json_list_data = jsc.textFile(_json_list_filename).cache();
 
-        JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir));
+        JavaRDD<String> json_ids = json_list_data.flatMap(new PagedJSON(_input_dir,_verbosity));
 
 
@@ -63 +63 @@
         System.out.println("");
         System.out.println("############");
-        System.out.println("# number of IDS: " + num_ids);
+        System.out.println("# Number of page ids: " + num_ids);
         System.out.println("############");
         System.out.println("");