Changeset 31011

Timestamp:

2016-10-31T15:40:36+13:00 (7 years ago)

Author:

davidb

Message:

Further RDD flatMap/map restructuring and refactoring, for per-page

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerPageJSONForeach.java (deleted)
• PerPageJSONMap.java (added)
• PerVolumeJSON.java (modified) (1 diff)
• ProcessForSolrIngest.java (modified) (2 diffs)
• __PerPageJSONForeach.java (added)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -1 +1 @@
 package org.hathitrust.extractedfeatures;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Set;
-
-import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.api.java.function.VoidFunction;
 import org.apache.spark.util.DoubleAccumulator;

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -119 +119 @@
         DoubleAccumulator progress_accum = jsc.sc().doubleAccumulator("Progress Percent");
         
-        PerPageJSONFlatmap paged_json = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
-        JavaRDD<JSONObject> json_per_page_ids = json_list_data.flatMap(paged_json).cache();
-        
-        PerPageJSONForeach paged_json_foreach = new PerPageJSONForeach(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
-        json_per_page_ids.foreach(paged_json_foreach);
+        PerPageJSONFlatmap paged_solr_json_flatmap = new PerPageJSONFlatmap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        JavaRDD<JSONObject> per_page_jsonobjects = json_list_data.flatMap(paged_solr_json_flatmap).cache();
+        
+        PerPageJSONMap paged_json_id_map = new PerPageJSONMap(_input_dir,_solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+        JavaRDD<String> per_page_ids = per_page_jsonobjects.map(paged_json_id_map);
 
 /*
@@ -133 +133 @@
 */
         
-        long num_page_ids = json_per_page_ids.count();
+        long num_page_ids = per_page_ids.count(); // trigger lazy eval of: flatmap:per-vol -> map:per-page
         
         System.out.println("");