Changeset 30951 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

Timestamp:

2016-10-26T17:54:44+13:00 (8 years ago)

Author:

davidb

Message:

Save a JSONObject as a file in the output directory

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (4 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -2 +2 @@
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import org.apache.commons.compress.compressors.CompressorException;
 import org.apache.spark.api.java.function.FlatMapFunction;
 import org.json.JSONArray;
@@ -85 +88 @@
         String json_dir = ClusterFileIO.removeSuffix(json_file_in,".json.bz2");
         String page_json_dir = json_dir + "/pages";
-        //ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
-        System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
+        ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
+        //System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
         
         ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
@@ -97 +100 @@
             }
             
-            // create JSON obj of just the page (for now)
-            // write it out
+            
             
             String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
-            
             ids.add(output_json_bz2);
             
@@ -107 +108 @@
                 System.out.println("Sample output JSON page file: " + output_json_bz2);
             }
+            
+            // create JSON obj of just the page (for now), and write it out
+            JSONObject ef_page = ef_pages.getJSONObject(i);
+            try {
+                BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(output_json_bz2);
+                bw.write(ef_page.toString());
+                bw.close();
+            } catch (IOException e) {
+                e.printStackTrace();
+            } catch (CompressorException e) {
+                e.printStackTrace();
+            }
+                        
+            
         }