Changeset 30945 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

Timestamp:

2016-10-26T15:37:24+13:00 (7 years ago)

Author:

davidb

Message:

Getting closer to writing out JSON files

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (5 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -24 +24 @@
 
     protected String _input_dir;
+    protected String _output_dir;
     protected int    _verbosity;
     
-    public PagedJSON(String input_dir, int verbosity)
+    public PagedJSON(String input_dir, String output_dir, int verbosity)
     {
-        _input_dir = input_dir;
-        _verbosity = verbosity;
+        _input_dir  = input_dir;
+        _output_dir = output_dir;
+        _verbosity  = verbosity;
     }
     
@@ -44 +46 @@
             while ((str = br.readLine()) != null) {
                 sb.append(str);
-                //System.out.println(str);
             }
 
             br.close();
-            
-            //System.err.println("*****" + sb.toString());
-                   
-            /*
-            List<String> lines = Files.readAllLines(path,StandardCharsets.UTF_8);
-
-
-            for (String line : lines) {
-                sb.append(line);
-
-            }
-            */
-            
         } 
         catch (Exception e) {
@@ -68 +56 @@
         JSONObject json_obj = new JSONObject(sb.toString());
         
-        
         return json_obj;
-        
-        //return sb.toString();
     }
 
@@ -83 +68 @@
         String id = extracted_feature_record.getString("id");
         
-        JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+        //JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
         JSONObject ef_features = extracted_feature_record.getJSONObject("features");
         
@@ -97 +82 @@
         int ef_num_pages = ef_pages.length();
         
+        // Make directory for page-level JSON output
+        String json_dir = ClusterFileIO.removeSuffix(json_file_in,".json.bz2");
+        String page_json_dir = json_dir + "/pages";
+        //ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
+        System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
+        
         ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
         for (int i = 0; i < ef_page_count; i++) {
-            ids.add(id + "." + i);
+            String formatted_i = String.format("page-%06d", i);
+            String page_id = id + "." + formatted_i;
+        
+            if (_verbosity >= 2) {
+              System.out.println("  Page: " + page_id);
+            }
+            
+            // create JSON obj of just the page (for now)
+            // write it out
+            
+            ids.add(page_json_dir +"/" + page_id + ".json.bz2");
+            
+            if (i==0) {
+                System.out.println("Sample output JSON page file: " + page_json_dir +"/" + page_id + ".json.bz2");
+            }
         }