Changeset 31013 for other-projects/hathitrust

Timestamp:

2016-10-31T20:51:39+13:00 (7 years ago)

Author:

davidb

Message:

Accumulator for PerPageMap

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerPageJSONMap.java (modified) (1 diff)
• ProcessForSolrIngest.java (modified) (1 diff)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONMap.java

@@ -64 +64 @@
         }
         
+        _progress_accum.add(_progress_step);
+        
         return output_json_bz2;
     }

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -117 +117 @@
         double per_vol = 100.0/(double)num_volumes;
         
-        DoubleAccumulator progress_accum = jsc.sc().doubleAccumulator("Progress Percent");
-        
-        PerPageJSONFlatmap paged_solr_json_flatmap = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        DoubleAccumulator per_vol_progress_accum = jsc.sc().doubleAccumulator("Per Volume Progress Percent");
+        
+        PerPageJSONFlatmap paged_solr_json_flatmap 
+            = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, per_vol_progress_accum,per_vol);
         JavaRDD<JSONObject> per_page_jsonobjects = json_list_data.flatMap(paged_solr_json_flatmap).cache();
         
-        PerPageJSONMap paged_json_id_map = new PerPageJSONMap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        //long num_page_ids = per_page_jsonobjects.count(); // trigger lazy eval of: flatmap:per-vol
+
+        DoubleAccumulator per_page_progress_accum = jsc.sc().doubleAccumulator("Pages Processed");
+        
+        PerPageJSONMap paged_json_id_map 
+            = new PerPageJSONMap(_input_dir,_solr_url,_output_dir,_verbosity, per_page_progress_accum,1.0);
         JavaRDD<String> per_page_ids = per_page_jsonobjects.map(paged_json_id_map);