Changeset 31252 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

Timestamp:

2016-12-20T14:15:05+13:00 (7 years ago)

Author:

davidb

Message:

Support for icu-tokenize property added, plus relevant refactoring.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java (modified) (1 diff)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/ProcessForWhitelist.java

@@ -72 +72 @@
         
         boolean strict_file_io = Boolean.getBoolean("wcsa-ef-ingest.strict-file-io");
-                
+        boolean icu_tokenize = Boolean.getBoolean("wcsa-ef-ingest.icu-tokenize");
+        
         PerVolumeWordStreamFlatmap paged_solr_wordfreq_flatmap 
             = new PerVolumeWordStreamFlatmap(_input_dir,_verbosity, 
                                      per_vol_progress_accum,per_vol,
+                                     icu_tokenize,
                                      strict_file_io);
         JavaRDD<String> words = json_list_data.flatMap(paged_solr_wordfreq_flatmap);