Changeset 30951

Timestamp:

2016-10-26T17:54:44+13:00 (6 years ago)

Author:

davidb

Message:

Save a JSONObject as a file in the output directory

Location:

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust

Files:

• PagedJSON.java (modified) (4 diffs)
• PrepareForIngest.java (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -2 +2 @@
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import org.apache.commons.compress.compressors.CompressorException;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.json.JSONArray;
@@ -85 +88 @@
         String json_dir = ClusterFileIO.removeSuffix(json_file_in,".json.bz2");
         String page_json_dir = json_dir + "/pages";
-        //ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
-        System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
+        ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
+        //System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
         
         ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
@@ -97 +100 @@
             }
             
-            // create JSON obj of just the page (for now)
-            // write it out
+            
             
             String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
-            
             ids.add(output_json_bz2);
             
@@ -107 +108 @@
                 System.out.println("Sample output JSON page file: " + output_json_bz2);
             }
+            
+            // create JSON obj of just the page (for now), and write it out
+            JSONObject ef_page = ef_pages.getJSONObject(i);
+            try {
+                BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(output_json_bz2);
+                bw.write(ef_page.toString());
+                bw.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            } catch (CompressorException e) {
+                e.printStackTrace();
+            }
+                        
+            
         }
         

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PrepareForIngest.java

@@ -45 +45 @@
         JavaRDD<String> json_ids = json_list_data.flatMap(paged_json).cache();
 
-        json_ids.saveAsTextFile("rdd-solr-json-page-files");
-
         long num_ids = json_ids.count();
         System.out.println("");
@@ -54 +52 @@
         System.out.println("");
 
+        System.out.println("############");
+        String rdd_save_file = "rdd-solr-json-page-files";
+        json_ids.saveAsTextFile(rdd_save_file);
+        System.out.println("# Saved RDD of Solr JSON page files, top-level, as:");
+        System.out.println("#  " + rdd_save_file);
+        System.out.println("############");
+        System.out.println("");
+        
         jsc.close();
     }