source: other-projects/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest/src/main/java/org/hathitrust/extractedfeatures/UniversalPOSLangMap.java@ 31506

package org.hathitrust.extractedfeatures;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class UniversalPOSLangMap 
{

    protected HashMap<String,HashMap<String,String>> _all_langmaps;
    
    protected HashMap<String,Integer> _missing_pos;
    
    public UniversalPOSLangMap(String langmap_directory) {
        System.out.println("Constructing: UniversalPOS Language Map");
        
        _missing_pos = new HashMap<String,Integer>();
        
        _all_langmaps = new HashMap<String,HashMap<String,String>>();
         
        List<Path> langmap_paths = null;
        
        URI langmap_directory_uri = null;
        
        try {
            langmap_directory_uri = new URI(langmap_directory);
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        try (Stream<Path> stream_paths = Files.walk(Paths.get(langmap_directory_uri))) {
            langmap_paths = stream_paths
                    .filter(Files::isRegularFile)
                    .collect(Collectors.toList());

        } catch (IOException e) {
            e.printStackTrace();
        }
        
        // For-each language file
        langmap_paths.forEach(langmap_path -> {
            File langmap_file = langmap_path.toFile();
            String lang_key = langmap_file.getName().substring(0,2);
            
            HashMap<String,String> pos_lookup = new HashMap<String,String>();
            
            // For-each line within that language file
            try (Stream<String> lang_lines = Files.lines(langmap_path)) {
                lang_lines.forEach(line -> {
                    String[] line_parts = line.split("\\t");
                    if (line_parts.length == 2) {
                        String pos_key = line_parts[0];
                        String pos_val = line_parts[1];
                        pos_lookup.put(pos_key, pos_val);
                    }
                });
            } catch (IOException e) {
                e.printStackTrace();
            }

            _all_langmaps.put(lang_key, pos_lookup);
        });

                System.out.println("Done Constructing UniversalPOS Language Map");
        
    }
    
    public int size()
    {
        return _all_langmaps.size();
    }
    
    public boolean containsLanguage(String lang_key)
    {
        return _all_langmaps.containsKey(lang_key);
    }

    public String getUniversalLanguagePOS(String lang_key,String opennlp_pos_key)
    {
        String universal_pos = null;
        
        HashMap<String,String> langmap = _all_langmaps.get(lang_key);
        if (langmap != null) {
            universal_pos = langmap.get(opennlp_pos_key);
            if (universal_pos == null) {
                String missing_lang_pos = lang_key + ":" + opennlp_pos_key;
                
                Integer mpos_freq = 0;
                if (_missing_pos.containsKey(missing_lang_pos)) {
                    mpos_freq = _missing_pos.get(missing_lang_pos);
                }
                else {
                    System.err.println("Warning: for language key '"+lang_key
                            +"' failed to find POS '" + opennlp_pos_key + "'");
                    System.err.println("Defaulting to POS 'X' (i.e., 'other')");
                }
                mpos_freq++;
                _missing_pos.put(lang_key,mpos_freq);
                
                universal_pos = "X";
            }
        }
        
        return universal_pos;
    } 
    
}