Changeset 31597

Timestamp:

2017-04-11T23:41:07+12:00 (7 years ago)

Author:

davidb

Message:

Additional _s and _ss fields to help with faceting. Temporarily commented out the full-text page part.

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• PerVolumeJSON.java (modified) (2 diffs)
• ProcessForSolrIngest.java (modified) (1 diff)
• SolrDocJSON.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -134 +134 @@
                     }
                 }
-                
+/*              
                 //
                 // Now move on to POS extracted features per-page
@@ -185 +185 @@
 
                 }
+                */
             }
         }

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -124 +124 @@
                                                        solr_endpoints,_output_dir,_verbosity,
                                                        icu_tokenize,strict_file_io);
-
         
         JavaRDD<Integer> per_volume_page_count = json_text_rdd.map(per_vol_json);
         
-        Integer num_page_ids = per_volume_page_count.reduce((a, b) -> a + b);
-        
-        System.out.println("");
-        System.out.println("############");
-        System.out.println("# Number of page ids: " + num_page_ids);
+        //Integer num_page_ids = per_volume_page_count.reduce((a, b) -> a + b);
+        long num_vol_ids = per_volume_page_count.count();
+        
+        System.out.println("");
+        System.out.println("############");
+        //System.out.println("# Number of page ids: " + num_page_ids);
+        System.out.println("# Number of volume ids: " + num_vol_ids);
         System.out.println("############");
         System.out.println("");

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -73 +73 @@
                 if (metavalue != null) {
                     solr_doc_json.put(metaname+"_t",metavalue);
+                    solr_doc_json.put(metaname+"_s",metavalue);
                 }
             }
@@ -80 +81 @@
                 if (metavalues != null) {
                     solr_doc_json.put(metaname+"_t",metavalues);
-                }
-            }
-            
-            solr_add_json.put("commitWithin", 5000);
+                    solr_doc_json.put(metaname+"_ss",metavalues);
+                }
+            }
+            
+            solr_add_json.put("commitWithin", 60000); // used to be 5000
             solr_add_json.put("doc", solr_doc_json);