Changeset 31011 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2016-10-31T15:40:36+13:00 (7 years ago)

Author:

davidb

Message:

Further RDD flatMap/map restructuring and refactoring, for per-page

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -119 +119 @@
         DoubleAccumulator progress_accum = jsc.sc().doubleAccumulator("Progress Percent");
         
-        PerPageJSONFlatmap paged_json = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
-        JavaRDD<JSONObject> json_per_page_ids = json_list_data.flatMap(paged_json).cache();
-        
-        PerPageJSONForeach paged_json_foreach = new PerPageJSONForeach(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
-        json_per_page_ids.foreach(paged_json_foreach);
+        PerPageJSONFlatmap paged_solr_json_flatmap = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        JavaRDD<JSONObject> per_page_jsonobjects = json_list_data.flatMap(paged_solr_json_flatmap).cache();
+        
+        PerPageJSONMap paged_json_id_map = new PerPageJSONMap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        JavaRDD<String> per_page_ids = per_page_jsonobjects.map(paged_json_id_map);
 
 /*
@@ -133 +133 @@
 */
         
-        long num_page_ids = json_per_page_ids.count();
+        long num_page_ids = per_page_ids.count(); // trigger lazy eval of: flatmap:per-vol -> map:per-page
         
         System.out.println("");