Changeset 31001 for other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/JSONSolrTransform.java

Timestamp:

2016-10-30T23:51:07+13:00 (7 years ago)

Author:

davidb

Message:

Code to work per-volume and per-page

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/JSONSolrTransform.java (modified) (2 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/extractedfeatures/JSONSolrTransform.java

@@ -1 +1 @@
 package org.hathitrust.extractedfeatures;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.Iterator;
 
+import org.apache.commons.compress.compressors.CompressorException;
 import org.json.JSONObject;
 
@@ -115 +123 @@
     }
 
+    public static void saveSolrDoc(JSONObject solr_add_doc_json, String output_file_json_bz2)
+    {
+        try {
+            BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(output_file_json_bz2);
+            bw.write(solr_add_doc_json.toString());
+            bw.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        } catch (CompressorException e) {
+            e.printStackTrace();
+        }
+    }
+    
+    public static void postSolrDoc(String post_url, JSONObject solr_add_doc_json)
+    {
+        
+        //String curl_popen = "curl -X POST -H 'Content-Type: application/json'";
+        //curl_popen += " 'http://10.11.0.53:8983/solr/htrc-pd-ef/update'";
+        //curl_popen += " --data-binary '";
+        //curl_popen += "'"
+
+
+        try {
+            HttpURLConnection httpcon = (HttpURLConnection) ((new URL(post_url).openConnection()));
+            httpcon.setDoOutput(true);
+            httpcon.setRequestProperty("Content-Type", "application/json");
+            httpcon.setRequestProperty("Accept", "application/json");
+            httpcon.setRequestMethod("POST");
+            httpcon.connect();
+
+            byte[] outputBytes = solr_add_doc_json.toString().getBytes("UTF-8");
+            OutputStream os = httpcon.getOutputStream();
+            os.write(outputBytes);
+            os.close();
+            
+            
+            // Read response
+            StringBuilder sb = new StringBuilder();
+            BufferedReader in = new BufferedReader(new InputStreamReader(httpcon.getInputStream()));
+            String decodedString;
+            while ((decodedString = in.readLine()) != null) {
+                sb.append(decodedString);
+            }
+            in.close();
+
+            JSONObject solr_status_json = new JSONObject(sb.toString());
+            JSONObject response_header_json = solr_status_json.getJSONObject("responseHeader");
+            if (response_header_json != null) {
+                int status = response_header_json.getInt("status");
+                if (status != 0) {
+                    System.err.println("Warning: POST request to " + post_url + " returned status " + status);
+                    System.err.println("Full response was: " + sb);
+                }
+            }
+            else {
+                System.err.println("Failed response to Solr POST: " + sb);
+            }
+            
+            
+            
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+    }
 }