Changeset 31509 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

Timestamp:

2017-03-13T20:50:06+13:00 (7 years ago)

Author:

davidb

Message:

LangPos determination changed to lock into first match, rather than trying to populate all model-predicted languages with the determined POS (which gets mixed up when l=en,de and POS=NE for example -- de goes wrong, has no match)

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java (modified) (6 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -19 +19 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
+
+import scala.Tuple2;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
@@ -388 +391 @@
             int text_len = text_al.size();
             
+            /*
             for (int li=0; li<lang_len; li++) {
                 String lang_key = lang_list[li];
@@ -393 +397 @@
                 if (universal_langmap.containsLanguage(lang_key))
                 {
+                */
                     HashMap<String,JSONArray> pos_lang_text_field_map = new HashMap<String,JSONArray>();
                     
@@ -405 +410 @@
                             String opennlp_pos_key = pos_tags[pti];
                             
-                            String upos = universal_langmap.getUniversalLanguagePOS(lang_key, opennlp_pos_key);
-                            String pos_lang_text_field = lang_key + "_" + upos + "_htrctoken";
+                            Tuple2<String,String> lang_pos_pair = universal_langmap.getUniversalLanguagePOSPair(lang_list, opennlp_pos_key);
+                            String selected_lang = lang_pos_pair._1;
+                            String upos = lang_pos_pair._2;
+                            
+                            String pos_lang_text_field = selected_lang;
+                            if (upos != null) {
+                                pos_lang_text_field += "_" + upos; 
+                            }
+                            pos_lang_text_field += "_htrctoken";
                             
                             if (!pos_lang_text_field_map.containsKey(pos_lang_text_field)) {
@@ -424 +436 @@
                         solr_doc_json.put(lang_text_field, json_values); 
                     }
+                    /*
                 }
                 else {
@@ -440 +453 @@
                 
             }
-            
+            */
         }
     }