Changeset 31001 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java

Timestamp:

2016-10-30T23:51:07+13:00 (7 years ago)

Author:

davidb

Message:

Code to work per-volume and per-page

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java (modified) (7 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/PagedJSON.java

@@ -9 +9 @@
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.Set;
 
@@ -28 +29 @@
 
 
-//class PagedJSON implements FlatMapFunction<String, String> 
-public class PagedJSON implements VoidFunction<String> 
+class PagedJSON implements FlatMapFunction<String, JSONObject>, VoidFunction<JSONObject> 
+//public class PagedJSON implements VoidFunction<String> 
 {
     private static final long serialVersionUID = 1L;
@@ -53 +54 @@
     }
     
-    public static void saveSolrDoc(JSONObject solr_add_doc_json, String output_file_json_bz2)
-    {
-        try {
-            BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(output_file_json_bz2);
-            bw.write(solr_add_doc_json.toString());
-            bw.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        } catch (CompressorException e) {
-            e.printStackTrace();
-        }
-    }
     
-    public static void postSolrDoc(String post_url, JSONObject solr_add_doc_json)
-    {
-        
-        //String curl_popen = "curl -X POST -H 'Content-Type: application/json'";
-        //curl_popen += " 'http://10.11.0.53:8983/solr/htrc-pd-ef/update'";
-        //curl_popen += " --data-binary '";
-        //curl_popen += "'"
-
-
-        try {
-            HttpURLConnection httpcon = (HttpURLConnection) ((new URL(post_url).openConnection()));
-            httpcon.setDoOutput(true);
-            httpcon.setRequestProperty("Content-Type", "application/json");
-            httpcon.setRequestProperty("Accept", "application/json");
-            httpcon.setRequestMethod("POST");
-            httpcon.connect();
-
-            byte[] outputBytes = solr_add_doc_json.toString().getBytes("UTF-8");
-            OutputStream os = httpcon.getOutputStream();
-            os.write(outputBytes);
-            os.close();
-            
-            
-            // Read response
-            StringBuilder sb = new StringBuilder();
-            BufferedReader in = new BufferedReader(new InputStreamReader(httpcon.getInputStream()));
-            String decodedString;
-            while ((decodedString = in.readLine()) != null) {
-                sb.append(decodedString);
-            }
-            in.close();
-
-            JSONObject solr_status_json = new JSONObject(sb.toString());
-            JSONObject response_header_json = solr_status_json.getJSONObject("responseHeader");
-            if (response_header_json != null) {
-                int status = response_header_json.getInt("status");
-                if (status != 0) {
-                    System.err.println("Warning: POST request to " + post_url + " returned status " + status);
-                    System.err.println("Full response was: " + sb);
-                }
-            }
-            else {
-                System.err.println("Failed response to Solr POST: " + sb);
-            }
-            
-            
-            
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-        }
-        
-    }
     
-    //public Iterator<String> call(String json_file_in) 
-    public void call(String json_file_in) 
+    public Iterator<JSONObject> call(String json_file_in) 
+    //public void call(String json_file_in) 
     { 
         JSONObject extracted_feature_record = JSONClusterFileIO.readJSONFile(_input_dir + "/" + json_file_in);
@@ -148 +84 @@
         ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
         
-        ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
+        ArrayList<JSONObject> json_pages = new ArrayList<JSONObject>(ef_num_pages);
         for (int i = 0; i < ef_page_count; i++) {
             String formatted_i = String.format("page-%06d", i);
@@ -158 +94 @@
             
             String output_json_bz2 = page_json_dir +"/" + formatted_i + ".json.bz2";
-            ids.add(output_json_bz2);
+            //ids.add(output_json_bz2);
             
             if (i==0) {
@@ -165 +101 @@
             
             JSONObject ef_page = ef_pages.getJSONObject(i);
-
+            
             if (ef_page != null) {
                 // Convert to Solr add form
                 JSONObject solr_add_doc_json = JSONSolrTransform.generateSolrDocJSON(volume_id, page_id, ef_page);
-
+                solr_add_doc_json.put("filename_json_bz2", output_json_bz2);
+                
+                json_pages.add(solr_add_doc_json);
             
-                if ((_verbosity >=2) && (i==20)) {
-                    System.out.println("==================");
-                    System.out.println("Sample output Solr add JSON [page 20]: " + solr_add_doc_json.toString());
-                    System.out.println("==================");
-                }
                 
-                            
-                if (_solr_url != null) {
-                    if ((_verbosity >=2) && (i==20)) {
-                        System.out.println("==================");
-                        System.out.println("Posting to: " + _solr_url);
-                        System.out.println("==================");
-                    }
-                    postSolrDoc(_solr_url, solr_add_doc_json);
-                }
-
-                if (_output_dir != null) {
-                    if ((_verbosity >=2) && (i==20)) {
-                        System.out.println("==================");
-                        System.out.println("Saving to: " + _output_dir);
-                        System.out.println("==================");
-                    }
-                    saveSolrDoc(solr_add_doc_json, _output_dir + "/" + output_json_bz2);
-                }
             }
             else {
@@ -203 +118 @@
         
         
-        ids.add(volume_id);
+        //ids.add(volume_id);
         _progress_accum.add(_progress_step);
         
         //return ids.iterator();
+        return json_pages.iterator();
     }
+    
+    public void call(JSONObject solr_add_doc_json) 
+    { 
+        String output_json_bz2 = solr_add_doc_json.getString("filename_json_bz2");
+        solr_add_doc_json.remove("filename_json_bz2");
+        
+        boolean random_test = (Math.random()>0.999); // every 1000
+        
+        if ((_verbosity >=2) && (random_test)) {
+            System.out.println("==================");
+            System.out.println("Sample output Solr add JSON [random test 1/1000]: " + solr_add_doc_json.toString());
+            System.out.println("==================");
+        }
+        
+                    
+        if (_solr_url != null) {
+            if ((_verbosity >=2) && (random_test)) {
+                System.out.println("==================");
+                System.out.println("Posting to: " + _solr_url);
+                System.out.println("==================");
+            }
+            JSONSolrTransform.postSolrDoc(_solr_url, solr_add_doc_json);
+        }
+
+        if (_output_dir != null) {
+            if ((_verbosity >=2) && (random_test)) {
+                System.out.println("==================");
+                System.out.println("Saving to: " + _output_dir);
+                System.out.println("==================");
+            }
+            JSONSolrTransform.saveSolrDoc(solr_add_doc_json, _output_dir + "/" + output_json_bz2);
+        }
+    }
+    
 }