Changeset 30970 for other-projects

Timestamp:

2016-10-28T11:10:32+13:00 (7 years ago)

Author:

davidb

Message:

Added in mapping of EF-JSON to Solr 'add' JSON format

File:

• other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java (modified) (7 diffs)

other-projects/hathitrust/solr-extracted-features/trunk/src/main/java/org/hathitrust/PagedJSON.java

@@ -4 +4 @@
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Set;
 
 import org.apache.commons.compress.compressors.CompressorException;
@@ -39 +43 @@
     protected JSONObject readJSONFile(String filename)
     {
-        //Path path = Paths.get(filename);
-
         StringBuilder sb = new StringBuilder();
 
@@ -59 +61 @@
         JSONObject json_obj = new JSONObject(sb.toString());
         
+        
         return json_obj;
     }
 
+    protected String generateSolrText(JSONObject ef_token_pos_count)
+    {
+        StringBuilder sb = new StringBuilder();
+
+        Iterator<String> token_iter = ef_token_pos_count.keys();
+        while (token_iter.hasNext()) {
+            String token = token_iter.next();
+            
+            sb.append(token);
+            if (token_iter.hasNext()) {
+                sb.append(" ");
+            }
+        }
+        
+        /*
+        Set<String> token_keys = ef_token_pos_count.keySet();
+        for (String token : token_keys) {
+                sb.append(token + " ");
+        }
+*/
+        
+       return sb.toString();
+    }
+    
+    protected JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page) 
+    {
+        JSONObject solr_add_json = null;
+        
+        if (ef_page != null) {
+            JSONObject ef_body = ef_page.getJSONObject("body");
+            if (ef_body != null) {
+                JSONObject ef_token_pos_count = ef_body.getJSONObject("tokenPosCount");
+                if (ef_token_pos_count != null) {
+
+                    solr_add_json = new JSONObject();
+
+                    String text = generateSolrText(ef_token_pos_count);
+        
+                    
+                    
+                    JSONObject solr_doc_json = new JSONObject();
+                    solr_doc_json.put("id", page_id);
+                    solr_doc_json.put("volumeid", volume_id);
+                    solr_doc_json.put("text_t", text);
+                    solr_doc_json.put("commitWithin", 5000);
+                   
+                    solr_add_json.put("add", solr_doc_json);
+                }
+                else {
+                    System.err.println("Warning: empty tokenPosCount field for '" + page_id + "'");
+                }
+            }
+            else {
+                System.err.println("Warning: empty body field for '" + page_id + "'");
+            }
+            
+        }
+        else {
+            System.err.println("Warning: null page for '" + page_id + "'");
+        }
+        
+        
+        /*
+          
+         /update/json/docs
+         */
+        
+        // For Reference ...
+        // Example documentation on Solr JSON syntax:
+        //   https://cwiki.apache.org/confluence/display/solr/Uploading+Data+with+Index+Handlers
+        //     #UploadingDatawithIndexHandlers-JSONFormattedIndexUpdates
+        
+        /*
+        curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/my_collection/update' --data-binary '
+        {
+          "add": {
+            "doc": {
+              "id": "DOC1",
+              "my_boosted_field": {         use a map with boost/value for a boosted field 
+                "boost": 2.3,
+                "value": "test"
+              },
+              "my_multivalued_field": [ "aaa", "bbb" ]    Can use an array for a multi-valued field 
+            }
+          },
+          "add": {
+            "commitWithin": 5000,           commit this document within 5 seconds 
+            "overwrite": false,             don't check for existing documents with the same uniqueKey 
+            "boost": 3.45,                  a document boost 
+            "doc": {
+              "f1": "v1",                   Can use repeated keys for a multi-valued field 
+              "f1": "v2"
+            }
+          },
+         
+          "commit": {},
+          "optimize": { "waitSearcher":false },
+         
+          "delete": { "id":"ID" },          delete by ID 
+          "delete": { "query":"QUERY" }     delete by query 
+        }'
+        */
+        
+        //return solr_doc_json;
+        return solr_add_json;
+    }
+    
+    protected void postSolrDoc(JSONObject solr_add_doc_json)
+    {
+        // "http://10.11.0.53:8983/solr/"
+        String post_url = "http://10.11.0.53:8983/solr/htrc-pd-ef/update";
+            
+        //String curl_popen = "curl -X POST -H 'Content-Type: application/json'";
+        //curl_popen += " 'http://10.11.0.53:8983/solr/htrc-pd-ef/update'";
+        //curl_popen += " --data-binary '";
+        //curl_popen += "'"
+
+
+        try {
+            HttpURLConnection httpcon = (HttpURLConnection) ((new URL(post_url).openConnection()));
+            httpcon.setDoOutput(true);
+            httpcon.setRequestProperty("Content-Type", "application/json");
+            httpcon.setRequestProperty("Accept", "application/json");
+            httpcon.setRequestMethod("POST");
+            httpcon.connect();
+
+            byte[] outputBytes = solr_add_doc_json.toString().getBytes("UTF-8");
+            OutputStream os = httpcon.getOutputStream();
+            os.write(outputBytes);
+            os.close();
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+    }
     public Iterator<String> call(String json_file_in) 
     { 
@@ -69 +208 @@
         
         
-        String id = extracted_feature_record.getString("id");
+        String volume_id = extracted_feature_record.getString("id");
         
         //JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+        //String title= ef_metadata.getString("title");
+        
         JSONObject ef_features = extracted_feature_record.getJSONObject("features");
         
@@ -89 +230 @@
         String page_json_dir = json_dir + "/pages";
         ClusterFileIO.createDirectoryAll(_output_dir + "/" + page_json_dir);
-        //System.out.println("mkdir: " + _output_dir + "/" + page_json_dir);
         
         ArrayList<String> ids = new ArrayList<String>(ef_num_pages);
         for (int i = 0; i < ef_page_count; i++) {
             String formatted_i = String.format("page-%06d", i);
-            String page_id = id + "." + formatted_i;
+            String page_id = volume_id + "." + formatted_i;
         
             if (_verbosity >= 2) {
@@ -107 +247 @@
             }
             
-            // create JSON obj of just the page (for now), and write it out
             JSONObject ef_page = ef_pages.getJSONObject(i);
-            try {
-                BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(_output_dir + "/" + output_json_bz2);
-                bw.write(ef_page.toString());
-                bw.close();
-            } catch (IOException e) {
-                e.printStackTrace();
-            } catch (CompressorException e) {
-                e.printStackTrace();
-            }
-                        
+
+            if (ef_page != null) {
+                // Convert to Solr add form
+                JSONObject solr_add_doc_json = generateSolrDocJSON(volume_id, page_id, ef_page);
+
+                if (i==20) {
+                    System.out.println("Sample output Solr add JSON [page 20]: " + solr_add_doc_json.toString());
+                    System.out.println("==================");
+                    //System.out.println("Sample text [page 20]: " + solr_add_doc_json.getString("text_t"));
+                }
+                
+                // create JSON obj of just the page (for now), and write it out
+                // write out the JSONOBject as a bz2 compressed file
+                /*
+                try {
+                    BufferedWriter bw = ClusterFileIO.getBufferedWriterForCompressedFile(_output_dir + "/" + output_json_bz2);
+                    bw.write(ef_page.toString());
+                    bw.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                } catch (CompressorException e) {
+                    e.printStackTrace();
+                }
+                */
+                                
+                //postSolrDoc(solr_add_doc_json);
+            
+            }
+            else {
+                System.err.println("Skipping: " + page_id);
+            }
             
         }
@@ -140 +300 @@
         
         
-        ids.add(id);
+        ids.add(volume_id);
         
         return ids.iterator();