Changeset 31252 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

Timestamp:

2016-12-20T14:15:05+13:00 (7 years ago)

Author:

davidb

Message:

Support for icu-tokenize property added, plus relevant refactoring.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForSolrIngest.java

@@ -121 +121 @@
         System.err.println();
         
+        boolean icu_tokenize = Boolean.getBoolean("wcsa-ef-ingest.icu-tokenize");
+        boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
+        
         PerVolumeJSON per_vol_json = new PerVolumeJSON(_input_dir,_whitelist_filename, 
-                                                       _solr_url,_output_dir,_verbosity, progress_accum,per_vol);
+                                                       _solr_url,_output_dir,_verbosity, progress_accum,per_vol,
+                                                       icu_tokenize,strict_file_io);
 
         json_list_data.foreach(per_vol_json);
@@ -161 +165 @@
         DoubleAccumulator per_vol_progress_accum = jsc.sc().doubleAccumulator("Per Volume Progress Percent");
         
-        //String strict_file_io_str = System.getProperty("wcsa-ef-ingest.strict-file-io","true");
+        boolean icu_tokenize = Boolean.getBoolean("wcsa-ef-ingest.icu-tokenize");
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
-                
+        
         PerPageJSONFlatmap paged_solr_json_flatmap 
             = new PerPageJSONFlatmap(_input_dir,_whitelist_filename,
                                      _solr_url,_output_dir,_verbosity, 
                                      per_vol_progress_accum,per_vol,
-                                     strict_file_io);
+                                     icu_tokenize,strict_file_io);
         JavaRDD<JSONObject> per_page_jsonobjects = json_list_data.flatMap(paged_solr_json_flatmap).cache();