Changeset 31505 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2017-03-13T15:31:40+13:00 (7 years ago)

Author:

davidb

Message:

Added in storing of top-level document metadata as separate solr-doc

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeJSON.java (modified) (2 diffs)
• SolrDocJSON.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -109 +109 @@
                 String volume_id = extracted_feature_record.getString("id");
 
-                //JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
+                JSONObject ef_metadata = extracted_feature_record.getJSONObject("metadata");
                 //String title= ef_metadata.getString("title");
 
+                //
+                // Top-level metadata Solr doc
+                //
+                JSONObject solr_add_metadata_doc_json = SolrDocJSON.generateToplevelMetadataSolrDocJSON(volume_id,ef_metadata);
+                if (solr_add_metadata_doc_json != null) {
+                
+                    if ((_verbosity >=2)) {
+                        System.out.println("==================");
+                        System.out.println("Metadata JSON: " + solr_add_metadata_doc_json.toString());
+                        System.out.println("==================");
+                    }
+
+                    if (solr_url != null) {
+
+                        if ((_verbosity >=2) ) {
+                            System.out.println("==================");
+                            System.out.println("Posting to: " + solr_url);
+                            System.out.println("==================");
+                        }
+                        SolrDocJSON.postSolrDoc(solr_url, solr_add_metadata_doc_json, volume_id, "top-level-metadata");
+                    }
+                }
+                
+                //
+                // Now move on to POS extracted features per-page
+                //
                 JSONObject ef_features = extracted_feature_record.getJSONObject("features");
 
@@ -150 +176 @@
 
                         if (solr_url != null) {
-                            if ((_verbosity >=2) && (i==20)) {
-                                System.out.println("==================");
-                                System.out.println("Posting to: " + solr_url);
-                                System.out.println("==================");
-                            }
                             SolrDocJSON.postSolrDoc(solr_url, solr_add_doc_json,
                                         volume_id, page_id);
                         }
-
-
                     }
                     else {

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -28 +28 @@
 public class SolrDocJSON {
 
+    protected static JSONObject generateToplevelMetadataSolrDocJSON(String volume_id, JSONObject ef_metadata)
+    {
+        JSONObject solr_update_json = null;
+        
+        
+        String [] metadata_single = new String[] {
+                "accessProfile",
+                "rightsAttributes",
+                "hathitrustRecordNumber",
+                "title",
+                "imprint",
+                "pubDate",
+                "pubPlace",
+                "language",
+                "issuance",
+                "typeOfResource"
+        };
+
+        String [] metadata_multiple = new String[] {
+                "oclc",
+                "isbn",
+                "issn",
+                "lccn",
+                "genre",        
+                "names"
+        };
+
+        if (ef_metadata != null) {
+            
+            // For JSON Solr format see:
+            //   https://cwiki.apache.org/confluence/display/solr/Uploading+Data+with+Index+Handlers
+                
+            //String title= ef_metadata.getString("title");
+            JSONObject solr_add_json = new JSONObject();
+            
+            JSONObject solr_doc_json = new JSONObject();
+            solr_doc_json.put("id", volume_id);
+            
+            for (String metaname: metadata_single) {
+                String metavalue = ef_metadata.getString(metaname);
+                if (metavalue != null) {
+                    solr_doc_json.put(metaname+"_t",metavalue);
+                }
+            }
+            
+            for (String metaname: metadata_multiple) {
+                JSONArray metavalues = ef_metadata.getJSONArray(metaname);
+                if (metavalues != null) {
+                    solr_doc_json.put(metaname+"_t",metavalues);
+                }
+            }
+            
+            solr_add_json.put("commitWithin", 5000);
+            solr_add_json.put("doc", solr_doc_json);
+
+            solr_update_json = new JSONObject();
+            solr_update_json.put("add",solr_add_json);
+        
+        }
+        else {
+            System.err.println("Warning: null metadata for '" + volume_id + "'");
+        }
+
+        return solr_update_json;
+    }
+
+
     
     protected static ArrayList<String> getTokenPosCountWords(JSONObject ef_token_pos_count, String page_id,
@@ -376 +443 @@
         }
     }
+    
     protected static JSONObject generateSolrDocJSON(String volume_id, String page_id, JSONObject ef_page,
                                                     WhitelistBloomFilter whitelist_bloomfilter, 
@@ -618 +686 @@
             e.printStackTrace();
         }
-        
     }
 }