Changeset 31503 for other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java

Timestamp:

2017-03-13T15:30:13+13:00 (7 years ago)

Author:

davidb

Message:

Monitor for missing POS keys, and print out details first time each missing type is encountered

File:

• other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java (modified) (2 diffs)

other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java

@@ -16 +16 @@
 
     protected HashMap<String,HashMap<String,String>> _all_langmaps;
+    
+    protected HashMap<String,Integer> _missing_pos;
     
     public UniversalPOSLangMap(String langmap_directory) {
@@ -86 +88 @@
         if (langmap != null) {
             universal_pos = langmap.get(opennlp_pos_key);
+            if (universal_pos == null) {
+                String missing_lang_pos = lang_key + ":" + opennlp_pos_key;
+                
+                Integer mpos_freq = 0;
+                if (_missing_pos.containsKey(missing_lang_pos)) {
+                    mpos_freq = _missing_pos.get(missing_lang_pos);
+                }
+                else {
+                    System.err.println("Warning: for language key '"+lang_key
+                            +"' failed to find POS '" + opennlp_pos_key + "'");
+                    System.err.println("Defaulting to POS 'X' (i.e., 'other')");
+                }
+                mpos_freq++;
+                _missing_pos.put(lang_key,mpos_freq);
+                
+                universal_pos = "X";
+            }
         }