Changeset 31252 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeWordStreamFlatmap.java

Timestamp:

2016-12-20T14:15:05+13:00 (7 years ago)

Author:

davidb

Message:

Support for icu-tokenize property added, plus relevant refactoring.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeWordStreamFlatmap.java (modified) (3 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeWordStreamFlatmap.java

@@ -20 +20 @@
     protected double            _progress_step;
     
+    boolean _icu_tokenize;
     boolean _strict_file_io;
     
     public PerVolumeWordStreamFlatmap(String input_dir, int verbosity, 
                               DoubleAccumulator progress_accum, double progress_step,
+                              boolean icu_tokenize,
                               boolean strict_file_io)
     {
@@ -32 +34 @@
         _progress_step  = progress_step;
         
+        _icu_tokenize   = icu_tokenize;
         _strict_file_io = strict_file_io;
     }
@@ -87 +90 @@
                 if (ef_page != null) {
                     
-                    ArrayList<String> page_word_list = SolrDocJSON.generateTokenPosCountText(volume_id, page_id, ef_page);                  
+                    ArrayList<String> page_word_list = SolrDocJSON.generateTokenPosCountText(volume_id, page_id, ef_page, _icu_tokenize);                   
                     all_word_list.addAll(page_word_list);
                 }