Changeset 31045 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

Timestamp:

2016-11-02T21:34:47+13:00 (7 years ago)

Author:

davidb

Message:

More careful treatment of what to do when a JSON file isn't there

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

@@ -1 +1 @@
 package org.hathitrust.extractedfeatures;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -31 +32 @@
     protected double            _progress_step;
     
+    boolean _strict_file_io;
+    
     public PerPageJSONFlatmap(String input_dir, String solr_url, String output_dir, int verbosity, 
-                     DoubleAccumulator progress_accum, double progress_step)
+                              DoubleAccumulator progress_accum, double progress_step,
+                              boolean strict_file_io)
     {
         _input_dir  = input_dir;
@@ -41 +45 @@
         _progress_accum = progress_accum;
         _progress_step  = progress_step;
+        
+        _strict_file_io = strict_file_io;
     }
     
-    public Iterator<JSONObject> call(String json_file_in) 
+    public Iterator<JSONObject> call(String json_file_in) throws IOException
     //public void call(String json_file_in) 
     { 
-        JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(_input_dir + "/" + json_file_in);
+        String full_json_file_in = _input_dir + "/" + json_file_in;
+        JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(full_json_file_in);
         
-        String volume_id = extracted_feature_record.getString("id");
+        ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>();
         
-        //JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
-        //String title= ef_metadata.getString("title");
-        
-        JSONObject ef_features = extracted_feature_record.getJSONObject("features");
-        
-        
-        int ef_page_count = ef_features.getInt("pageCount");
-        
-        if (_verbosity >= 1) {
-            System.out.println("Processing: " + json_file_in);
-            System.out.println("  pageCount = " + ef_page_count);
+        if (extracted_feature_record != null) {
+            String volume_id = extracted_feature_record.getString("id");
+
+            //JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+            //String title= ef_metadata.getString("title");
+
+            JSONObject ef_features = extracted_feature_record.getJSONObject("features");
+
+
+            int ef_page_count = ef_features.getInt("pageCount");
+
+            if (_verbosity >= 1) {
+                System.out.println("Processing: " + json_file_in);
+                System.out.println("  pageCount = " + ef_page_count);
+            }
+
+            JSONArray ef_pages = ef_features.getJSONArray("pages");
+            int ef_num_pages = ef_pages.length();
+            if (ef_num_pages != ef_page_count) {
+                System.err.println("Warning: number of page elements in JSON (" + ef_num_pages + ")"
+                        +" does not match 'pageCount' metadata (" + ef_page_count + ")"); 
+            }
+
+            // Make directory for page-level JSON output
+            String json_dir = ClusterFileIO.removeSuffix(json_file_in,".json.bz2");
+            String page_json_dir = json_dir + "/pages";
+            ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
+
+            if (_verbosity >= 2) {
+                System.out.print("  Pages: ");
+            }
+
+            for (int i = 0; i < ef_page_count; i++) {
+                String formatted_i = String.format("page-%06d", i);
+                String page_id = volume_id + "." + formatted_i;
+
+                if (_verbosity >= 2) {
+                    if (i>0) {
+                        System.out.print(", ");
+                    }
+                    System.out.print(page_id);
+                }
+
+                String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
+                
+                if (i==(ef_page_count-1)) {
+                    if (_verbosity >= 2) {
+                        System.out.println();
+                    }
+                    System.out.println("Sample output JSON page file: " + output_json_bz2);
+                }
+
+                JSONObject ef_page = ef_pages.getJSONObject(i);
+
+                if (ef_page != null) {
+                    // Convert to Solr add form
+                    JSONObject solr_add_doc_json = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page);
+                    solr_add_doc_json.put("filename_json_bz2", output_json_bz2);
+
+                    json_pages.add(solr_add_doc_json);
+
+
+                }
+                else {
+                    System.err.println("Skipping: " + page_id);
+                }
+
+            }
         }
-        
-        JSONArray ef_pages = ef_features.getJSONArray("pages");
-        int ef_num_pages = ef_pages.length();
-        
-        // Make directory for page-level JSON output
-        String json_dir = ClusterFileIO.removeSuffix(json_file_in,".json.bz2");
-        String page_json_dir = json_dir + "/pages";
-        ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
-        
-        if (_verbosity >= 2) {
-              System.out.print("  Pages: ");
-            }
-        
-        ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>(ef_num_pages);
-        for (int i = 0; i < ef_page_count; i++) {
-            String formatted_i = String.format("page-%06d", i);
-            String page_id = volume_id + "." + formatted_i;
-        
-            if (_verbosity >= 2) {
-              if (i>0) {
-                  System.out.print(", ");
-              }
-              System.out.print(page_id);
-            }
-            
-            String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
-            //ids.add(output_json_bz2);
-            
-            if (i==(ef_page_count-1)) {
-                if (_verbosity >= 2) {
-                    System.out.println();
-                }
-                System.out.println("Sample output JSON page file: " + output_json_bz2);
-            }
-            
-            JSONObject ef_page = ef_pages.getJSONObject(i);
-            
-            if (ef_page != null) {
-                // Convert to Solr add form
-                JSONObject solr_add_doc_json = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page);
-                solr_add_doc_json.put("filename_json_bz2", output_json_bz2);
-                
-                json_pages.add(solr_add_doc_json);
-            
-                
+        else {
+            // File did not exist, or could not be parsed
+            String mess = "Failed to read in bzipped JSON file '" + full_json_file_in + "'";
+            if (_strict_file_io) {
+                throw new IOException(mess);
             }
             else {
-                System.err.println("Skipping: " + page_id);
+                System.err.println("Warning: " + mess);
+                System.out.println("Warning: " + mess);
             }
-            
         }
-        
-        
-        //ids.add(volume_id);
+
         _progress_accum.add(_progress_step);
         
-        //return ids.iterator();
         return json_pages.iterator();
     }