Changeset 31375 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

Timestamp:

2017-01-31T21:35:50+13:00 (7 years ago)

Author:

davidb

Message:

Initial cut at including POS information to solr index

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java (modified) (7 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/PerVolumeJSON.java

@@ -3 +3 @@
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 
@@ -29 +30 @@
     protected String _input_dir;
     protected String _whitelist_filename;
+    protected String _langmap_directory;
     
     protected String _solr_url;
@@ -36 +38 @@
     
     protected WhitelistBloomFilter _whitelist_bloomfilter;
-    
-
-    
-     boolean _icu_tokenize;
-     boolean _strict_file_io;
-     
-    public PerVolumeJSON(String input_dir, String whitelist_filename,
+    protected UniversalPOSLangMap _universal_langmap;
+
+    boolean _icu_tokenize;
+    boolean _strict_file_io;
+
+    public PerVolumeJSON(String input_dir, String whitelist_filename, String langmap_directcory,
                          String solr_url, String output_dir, int verbosity, 
                          boolean icu_tokenize, boolean strict_file_io)
@@ -48 +49 @@
         _input_dir  = input_dir;
         _whitelist_filename = whitelist_filename;
+        _langmap_directory = langmap_directcory;
         
         _solr_url   = solr_url;
@@ -57 +59 @@
         
         _whitelist_bloomfilter = null;
+        _universal_langmap = null;
     }
     
@@ -66 +69 @@
             _whitelist_bloomfilter = new WhitelistBloomFilter(_whitelist_filename,true);
         }
-
+        
+        if ((_langmap_directory != null) && (_universal_langmap == null)) {
+            _universal_langmap = new UniversalPOSLangMap(_langmap_directory);
+        }
         int ef_num_pages = 0;
 
@@ -107 +113 @@
                         // Convert to Solr add form
                         JSONObject solr_add_doc_json 
-                        = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _icu_tokenize);
+                        = SolrDocJSON.generateSolrDocJSON(volume_id, page_id, ef_page, _whitelist_bloomfilter, _universal_langmap, _icu_tokenize);