Changeset 31266 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

Timestamp:

2016-12-27T18:51:42+13:00 (7 years ago)

Author:

davidb

Message:

Rekindling of per-volume approach. Also some tweaking to verbosity debug printing in per-page

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java (modified) (7 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -2 +2 @@
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
 
+import org.apache.spark.api.java.function.FlatMapFunction;
 import org.apache.spark.api.java.function.VoidFunction;
 import org.apache.spark.util.DoubleAccumulator;
@@ -18 +21 @@
 
 
-public class PerVolumeJSON implements VoidFunction<String> 
+//public class PerVolumeJSON implements VoidFunction<String> 
+public class PerVolumeJSON implements FlatMapFunction<String,String> 
 {
     private static final long serialVersionUID = 1L;
@@ -58 +62 @@
     }
     
-    //public Iterator<String> call(String json_file_in) 
-    public void call(String json_file_in) throws IOException
+    //public void call(String json_file_in) throws IOException
+    public Iterator<String> call(String json_file_in) throws IOException
+    
     { 
         if ((_whitelist_filename != null) && (_whitelist_bloomfilter == null)) {
@@ -65 +70 @@
         }
 
+        ArrayList<String> ids = null;
+        
         String full_json_file_in = _input_dir + "/" + json_file_in;
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
@@ -94 +101 @@
             }
 
-            //ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
+            ids = new ArrayList<String>(ef_num_pages);
             for (int i = 0; i < ef_page_count; i++) {
                 String formatted_i = String.format("page-%06d", i);
@@ -104 +111 @@
 
                 String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
-                //ids.add(output_json_bz2); // ****
+                ids.add(page_id); 
 
                 if (i==0) {
@@ -164 +171 @@
         _progress_accum.add(_progress_step);
         
-        //return ids.iterator();
+        return ids.iterator();
     }
 }