Changeset 31252 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

Timestamp:

2016-12-20T14:15:05+13:00 (7 years ago)

Author:

davidb

Message:

Support for icu-tokenize property added, plus relevant refactoring.

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerPageJSONFlatmap.java

@@ -38 +38 @@
     protected double            _progress_step;
     
+    boolean _icu_tokenize;
     boolean _strict_file_io;
     
@@ -43 +44 @@
                               String solr_url, String output_dir, int verbosity, 
                               DoubleAccumulator progress_accum, double progress_step,
-                              boolean strict_file_io)
+                              boolean icu_tokenize, boolean strict_file_io)
     {
         _input_dir  = input_dir;
@@ -55 +56 @@
         _progress_step  = progress_step;
         
+        _icu_tokenize   = icu_tokenize;
         _strict_file_io = strict_file_io;
         
@@ -132 +134 @@
                     // Convert to Solr add form
                     JSONObject solr_add_doc_json 
-                        = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter);
+                        = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter,_icu_tokenize);
                     solr_add_doc_json.put("filename_json_bz2", output_json_bz2);