Changeset 31509 for other-projects

Timestamp:

2017-03-13T20:50:06+13:00 (7 years ago)

Author:

davidb

Message:

LangPos determination changed to lock into first match, rather than trying to populate all model-predicted languages with the determined POS (which gets mixed up when l=en,de and POS=NE for example -- de goes wrong, has no match)

Location:

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures

Files:

• SolrDocJSON.java (modified) (6 diffs)
• UniversalPOSLangMap.java (modified) (4 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/SolrDocJSON.java

@@ -19 +19 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
+
+import scala.Tuple2;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
@@ -388 +391 @@
             int text_len = text_al.size();
             
+            /*
             for (int li=0; li<lang_len; li++) {
                 String lang_key = lang_list[li];
@@ -393 +397 @@
                 if (universal_langmap.containsLanguage(lang_key))
                 {
+                */
                     HashMap<String,JSONArray> pos_lang_text_field_map = new HashMap<String,JSONArray>();
                     
@@ -405 +410 @@
                             String opennlp_pos_key = pos_tags[pti];
                             
-                            String upos = universal_langmap.getUniversalLanguagePOS(lang_key, opennlp_pos_key);
-                            String pos_lang_text_field = lang_key + "_" + upos + "_htrctoken";
+                            Tuple2<String,String> lang_pos_pair = universal_langmap.getUniversalLanguagePOSPair(lang_list, opennlp_pos_key);
+                            String selected_lang = lang_pos_pair._1;
+                            String upos = lang_pos_pair._2;
+                            
+                            String pos_lang_text_field = selected_lang;
+                            if (upos != null) {
+                                pos_lang_text_field += "_" + upos; 
+                            }
+                            pos_lang_text_field += "_htrctoken";
                             
                             if (!pos_lang_text_field_map.containsKey(pos_lang_text_field)) {
@@ -424 +436 @@
                         solr_doc_json.put(lang_text_field, json_values); 
                     }
+                    /*
                 }
                 else {
@@ -440 +453 @@
                 
             }
-            
+            */
         }
     }

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java

@@ -11 +11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+
+import scala.Tuple2;
 
 public class UniversalPOSLangMap 
@@ -83 +85 @@
     }
 
-    public String getUniversalLanguagePOS(String lang_key,String opennlp_pos_key)
+    public String getUniversalLanguagePOSUnchecked(String lang_key,String opennlp_pos_key)
     {
         String universal_pos = null;
@@ -90 +92 @@
         if (langmap != null) {
             universal_pos = langmap.get(opennlp_pos_key);
-            if (universal_pos == null) {
-                String missing_lang_pos = lang_key + ":" + opennlp_pos_key;
-                
-                Integer mpos_freq = 0;
-                if (_missing_pos.containsKey(missing_lang_pos)) {
-                    mpos_freq = _missing_pos.get(missing_lang_pos);
-                }
-                else {
-                    System.err.println("Warning: for language key '"+lang_key
-                            +"' failed to find POS '" + opennlp_pos_key + "'");
-                    System.err.println("Defaulting to POS 'X' (i.e., 'other')");
-                }
-                mpos_freq++;
-                _missing_pos.put(lang_key,mpos_freq);
-                
-                universal_pos = "X";
-            }
         }
         
@@ -112 +97 @@
     } 
     
+    public String getUniversalLanguagePOSChecked(String lang_key,String opennlp_pos_key)
+    {
+        if (!_all_langmaps.containsKey(lang_key)) {
+            // Not a language with a POS map
+            return "";
+        }
+        
+        String universal_pos = null;
+        
+        HashMap<String,String> langmap = _all_langmaps.get(lang_key);
+        universal_pos = langmap.get(opennlp_pos_key);
+        
+        if (universal_pos == null) {
+            String missing_lang_pos = lang_key + ":" + opennlp_pos_key;
+
+            Integer mpos_freq = 0;
+            if (_missing_pos.containsKey(missing_lang_pos)) {
+                mpos_freq = _missing_pos.get(missing_lang_pos);
+            }
+            else {
+                System.err.println("Warning: for language key '"+lang_key
+                        +"' failed to find POS '" + opennlp_pos_key + "'");
+                System.err.println("Defaulting to POS 'X' (i.e., 'other')");
+            }
+            mpos_freq++;
+            _missing_pos.put(missing_lang_pos,mpos_freq);
+
+            universal_pos = "X";
+        }
+        
+        return universal_pos;
+    } 
+    
+    public Tuple2<String,String> getUniversalLanguagePOSPair(String[] lang_keys,String opennlp_pos_key)
+    {
+        String universal_pos = null;
+        String selected_lang = null;
+        
+        for (int li=0; li<lang_keys.length; li++) {
+            String lang_key = lang_keys[li];
+            
+            universal_pos = getUniversalLanguagePOSUnchecked(lang_key,opennlp_pos_key);
+            if (universal_pos != null) {
+                selected_lang = lang_key;
+                break;
+            }
+        }
+    
+        if (universal_pos == null) {
+            // Failed to any match in any of the given languages
+            // => Lock onto the first language (highest probability when modeled)
+            selected_lang = lang_keys[0];
+            
+            if (!_all_langmaps.containsKey(selected_lang)) {
+                // Not a language with a POS map
+                return new Tuple2<String,String>(selected_lang,null);
+            }
+            
+            // If here, then is a POS language => default to "X" 
+            
+            String missing_lang_pos = selected_lang + ":" + opennlp_pos_key;
+
+            Integer mpos_freq = 0;
+            if (_missing_pos.containsKey(missing_lang_pos)) {
+                mpos_freq = _missing_pos.get(missing_lang_pos);
+            }
+            else {
+                System.err.println("Warning: for language key '"+selected_lang
+                        +"' failed to find POS '" + opennlp_pos_key + "'");
+                System.err.println("Defaulting to POS 'X' (i.e., 'other')");
+            }
+            mpos_freq++;
+            _missing_pos.put(missing_lang_pos,mpos_freq);
+
+            universal_pos = "X";
+        }
+
+        return new Tuple2<String,String>(selected_lang,universal_pos);
+    } 
 }