Changeset 31675 for other-projects/hathitrust/wcsa/extracted-features-solr

Timestamp:

2017-05-11T22:19:06+12:00 (7 years ago)

Author:

davidb

Message:

More careful set of metadata fields indexed

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeJSON.java (modified) (1 diff)
• SolrDocJSON.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -134 +134 @@
                     }
                 }
-/*              
+                
                 //
                 // Now move on to POS extracted features per-page
                 //
-                JSONObject ef_features = extracted_feature_record.getJSONObject("features");
-
-                int ef_page_count = ef_features.getInt("pageCount");
-
-                if (_verbosity >= 1) {
-                    System.out.println("Processing: " + volume_id);
-                    System.out.println("  pageCount = " + ef_page_count);
-                }
-
-                JSONArray ef_pages = ef_features.getJSONArray("pages");
-                ef_num_pages = ef_pages.length();
-
-
-                for (int i = 0; i < ef_page_count; i++) {
-                    String formatted_i = String.format("page-%06d", i);
-                    String page_id = volume_id + "." + formatted_i;
-
-                    if (_verbosity >= 2) {
-                        System.out.println("  Page: " + page_id);
-                    }
-
-
-                    JSONObject ef_page = ef_pages.getJSONObject(i);
-
-                    if (ef_page != null) {
-                        // Convert to Solr add form
-                        JSONObject solr_add_doc_json 
-                        = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _universal_langmap, _icu_tokenize);
-
-
-                        if ((_verbosity >=2) && (i==20)) {
-                            System.out.println("==================");
-                            System.out.println("Sample output Solr add JSON [page 20]: " + solr_add_doc_json.toString());
-                            System.out.println("==================");
-                        }
-
-
-                        if (solr_url != null) {
-                            SolrDocJSON.postSolrDoc(solr_url, solr_add_doc_json,
+                boolean index_pages = true;
+                if (index_pages) {
+                    
+                    JSONObject ef_features = extracted_feature_record.getJSONObject("features");
+
+                    int ef_page_count = ef_features.getInt("pageCount");
+
+                    if (_verbosity >= 1) {
+                        System.out.println("Processing: " + volume_id);
+                        System.out.println("  pageCount = " + ef_page_count);
+                    }
+
+                    JSONArray ef_pages = ef_features.getJSONArray("pages");
+                    ef_num_pages = ef_pages.length();
+
+
+                    for (int i = 0; i < ef_page_count; i++) {
+                        String formatted_i = String.format("page-%06d", i);
+                        String page_id = volume_id + "." + formatted_i;
+
+                        if (_verbosity >= 2) {
+                            System.out.println("  Page: " + page_id);
+                        }
+
+
+                        JSONObject ef_page = ef_pages.getJSONObject(i);
+
+                        if (ef_page != null) {
+                            // Convert to Solr add form
+                            JSONObject solr_add_doc_json 
+                            = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _universal_langmap, _icu_tokenize);
+
+
+                            if ((_verbosity >=2) && (i==20)) {
+                                System.out.println("==================");
+                                System.out.println("Sample output Solr add JSON [page 20]: " + solr_add_doc_json.toString());
+                                System.out.println("==================");
+                            }
+
+
+                            if (solr_url != null) {
+                                SolrDocJSON.postSolrDoc(solr_url, solr_add_doc_json,
                                         volume_id, page_id);
-                        }
-                    }
-                    else {
-                        System.err.println("Skipping: " + page_id);
-                    }
-
-                }
-                */
+                            }
+                        }
+                        else {
+                            System.err.println("Skipping: " + page_id);
+                        }
+
+                    }
+                }
             }
         }

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -34 +34 @@
     {
         JSONObject solr_update_json = null;
-        
+        /*
+         Example JSON for id: "gri.ark:/13960/t0003qw46
+         metadata: {
+         
+        "accessProfile": "open",
+        "bibliographicFormat": "BK",
+        "classification": {
+            "lcc": [
+                "ND646 .B8 1900"
+            ]
+        },
+        "dateCreated": "2016-06-19T08:30:16.11199Z",
+        "enumerationChronology": " ",
+        "genre": [
+            "not fiction"
+        ],
+        "governmentDocument": false,
+        "handleUrl": "http://hdl.handle.net/2027/gri.ark:/13960/t0003qw46",
+        "hathitrustRecordNumber": "100789562",
+        "htBibUrl": "http://catalog.hathitrust.org/api/volumes/full/htid/gri.ark:/13960/t0003qw46.json",
+        "imprint": "Burlington Fine Arts Club, 1900.",
+        "isbn": [],
+        "issn": [],
+        "issuance": "monographic",
+        "language": "eng",
+        "lastUpdateDate": "2015-09-14 13:25:03",
+        "lccn": [],
+        "names": [
+            "Burlington Fine Arts Club "
+        ],
+        "oclc": [
+            "25259734"
+        ],
+        "pubDate": "1900",
+        "pubPlace": "enk",
+        "rightsAttributes": "pd",
+        "schemaVersion": "1.3",
+        "sourceInstitution": "CMALG",
+        "sourceInstitutionRecordNumber": "9928077890001551",
+        "title": "Exhibition of pictures by Dutch masters of the seventeenth century.",
+        "typeOfResource": "text",
+        "volumeIdentifier": "gri.ark:/13960/t0003qw46"
+
+          }
+         
+         */
         
         String [] metadata_single = new String[] {
                 "accessProfile",
-                "rightsAttributes",
-                "hathitrustRecordNumber",
-                "title",
+                "bibliographicFormat",
+                "dateCreated",              // date
+                //"enumerationChronology",  // What is this?
+                "governmentDocument",       // bool: true/false
+                "handleUrl",
+                "hathitrustRecordNumber",   // int?
+                "htBibUrl",
                 "imprint",
+                "issuance",
+                "language",
+                "lastUpdateDate",
                 "pubDate",
                 "pubPlace",
-                "language",
-                "issuance",
-                "typeOfResource"
+                "rightsAttributes",
+                "schemaVersion",
+                "sourceInstitution",
+                "sourceInstitutionRecordNumber",
+                "title",
+                "typeOfResource",
+                "volumeIdentifier"
         };
 
@@ -58 +114 @@
         };
 
+        String [] metadata_hashmap_multiple = new String[] {
+                "classification"
+        };
+        
+        
         if (ef_metadata != null) {
             
@@ -82 +143 @@
                     solr_doc_json.put(metaname+"_t",metavalues);
                     solr_doc_json.put(metaname+"_ss",metavalues);
+                }
+            }
+            
+            for (String metaname: metadata_hashmap_multiple) {
+                JSONObject metakeys = ef_metadata.getJSONObject(metaname);
+                
+                if (metakeys != null) {
+                    Iterator<String> metakey_iter = metakeys.keys();
+                    while (metakey_iter.hasNext()) {
+                        String metakey = metakey_iter.next();
+                        
+                        JSONArray metavalues = metakeys.getJSONArray(metakey);
+                        if (metavalues != null) {
+                            String combined_metaname = metaname + "_" + metakey;
+                            solr_doc_json.put(combined_metaname+"_t",metavalues);
+                            solr_doc_json.put(combined_metaname+"_ss",metavalues);
+                        }   
+                    }
                 }
             }