source: other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java@ 33674

/**
 * Class that uses OpenNLP with the Language Detection Model to determine, with a default
 * or configurable level of confidence, whether text (from a file or stdin) is in a given
 * language or not.
 * Internal functions can be used for detecting any of the 103 languages currently supported by
 * the OpenNLP Language Detection Model.
 * 
 * http://opennlp.apache.org/news/model-langdetect-183.html
 * language detector model: http://opennlp.apache.org/models.html
 *        Pre-trained models for OpenNLP 1.5: http://opennlp.sourceforge.net/models-1.5/
 * Use of Apache OpenNLP in general:
 *   http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#intro.api
 * Use of OpenNLP for language detection:
 * http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#tools.langdetect
 * 
 * This code was based on the information and sample code at the above links and the links dispersed throughout this file.
 * See also the accompanying README file.
 *
 * July 2019
 */

package org.greenstone.atea;

import java.io.*;
import opennlp.tools.langdetect.*;
import opennlp.tools.sentdetect.*;
import opennlp.tools.util.*;

import java.util.ArrayList;

import org.greenstone.atea.morphia.SentenceInfo;
import org.greenstone.atea.morphia.LanguageInfo;

/**
 * EXPORT OPENNLP_HOME environment variable to be your apache OpenNLP installation.
 * Create a folder called "models" within the $OPENNLP_HOME folder, and put the file "langdetect-183.bin" in there
 *    (which is the language detection model zipped up and renamed to .bin extension).
 *
 * Then, to compile this program, do the following from the "src" folder (the folder containing this java file):
 *    maori-lang-detection/src$ javac -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/TextLanguageDetector.java
 *
 * Only the subclass MaoriTextDetector.java has a main method at present that can be run.
 *
 */
public class TextLanguageDetector {

    public static final double DEFAULT_MINIMUM_CONFIDENCE = 0.50;

    /**
     * Configurable: cut off minimum confidence value,
     * greater or equal to which determines that the best predicted language is
     * acceptable to user of TextLanguageDetector.
     */
    public final double MINIMUM_CONFIDENCE;

    /** Number of language and confidence results to return for storing in MongoDB
     * MongoDB runs out of space if storing too many, as we store this info per sentence
     * and a long text document becomes a very large MongoDB document presumable*/
    public final int NUM_TOP_LANGUAGES = 5; // 103 max, in current version of opennlp lang model
    
    /** silentMode set to false means TextLanguageDetector won't print helpful messages while running. Set to true to run silently. */
    public final boolean silentMode;

    private final String OPENNLP_MODELS_RELATIVE_PATH = "models" + File.separator; 
    
    /** Language Detection Model file for OpenNLP is expected to be at $OPENNLP_HOME/models/langdetect-183.bin */
    private final String LANG_DETECT_MODEL_RELATIVE_PATH = OPENNLP_MODELS_RELATIVE_PATH + "langdetect-183.bin";

    /**
     * The LanguageDetectorModel object that will do the actual language detection/prediction for us.
     * Created once in the constructor, can be used as often as needed thereafter.
    */
    private LanguageDetector myCategorizer = null;

    /** 
     * The Sentence Detection object that does the sentence splitting for the language
     * the sentece model was trained for.
     */
    private SentenceDetectorME sentenceDetector = null;


    /** Constructor with default confidence for language detection.
     * Does not create sentence model, just the language detection model.
     */
    public TextLanguageDetector(boolean silentMode) throws Exception {
    this(silentMode, DEFAULT_MINIMUM_CONFIDENCE);
    }
    
    /** Constructor with configurable min_confidence for language detection
     * Does not create sentence model, just the language detection model.
     */
    public TextLanguageDetector(boolean silentMode, double min_confidence) throws Exception {
    this.silentMode = silentMode;
    this.MINIMUM_CONFIDENCE = min_confidence;

    // 1. Check we can find the Language Detect Model file in the correct location (check that $OPENNLP_HOME/models/langdetect-183.bin exists);
    String langDetectModelPath = System.getenv("OPENNLP_HOME");
    if(System.getenv("OPENNLP_HOME") == null) {
        throw new Exception("\n\t*** Environment variable OPENNLP_HOME must be set to your Apache OpenNLP installation folder.");
    }   
    langDetectModelPath = langDetectModelPath + File.separator + LANG_DETECT_MODEL_RELATIVE_PATH;
    File langDetectModelBinFile = new File(langDetectModelPath);
    if(!langDetectModelBinFile.exists()) {
        throw new Exception("\n\t*** " + langDetectModelBinFile.getPath() + " doesn't exist."
                + "\n\t*** Ensure the $OPENNLP_HOME folder contains a 'models' folder"
                + "\n\t*** with the model file 'langdetect-183.bin' in it.");
    }


    // 2. Set up our language detector Model and the Categorizer for language predictions based on the Model.
    // http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#intro.api
    // https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
    try (InputStream modelIn = new FileInputStream(langDetectModelPath)) {

        LanguageDetectorModel model = new LanguageDetectorModel(modelIn);

        // http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#tools.langdetect
        this.myCategorizer = new LanguageDetectorME(model);
    }/*catch(Exception e) {
        e.printStackTrace();
        }*/
    
    // instantiating function should handle critical exceptions. Constructors shouldn't.
    
    }
    
    /** More general constructor that additionally can load up the sentence detector model
     * for other languages, as long as the provided trained sentence model .bin file exists 
     * in the OPENNLP_MODELS_RELATIVE_PATH folder. */
    public TextLanguageDetector(boolean silentMode, double min_confidence,
                 String sentenceModelFileName) throws Exception
    {
    this(silentMode, min_confidence);

    // 3. Set up our sentence model and SentenceDetector object
    String sentenceModelPath = System.getenv("OPENNLP_HOME") + File.separator
        + OPENNLP_MODELS_RELATIVE_PATH + sentenceModelFileName; // "mri-sent_trained.bin" default
    File sentenceModelBinFile = new File(sentenceModelPath);
    if(!sentenceModelBinFile.exists()) {        
        throw new Exception("\n\t*** " + sentenceModelBinFile.getPath() + " doesn't exist."
                + "\n\t*** Ensure the $OPENNLP_HOME folder contains a 'models' folder"
                + "\n\t*** with the model file "+sentenceModelFileName+" in it.");
    }
    try (InputStream modelIn = new FileInputStream(sentenceModelPath)) {
        // https://www.tutorialspoint.com/opennlp/opennlp_sentence_detection.htm
        SentenceModel sentenceModel = new SentenceModel(modelIn);       
        this.sentenceDetector = new SentenceDetectorME(sentenceModel);
        
    } // instantiating function should handle this critical exception
    }

    /** TODO: Is it sensible to use the Maori Language Sentence Model to split the text
     * into sentences? What if the text in any other language or a mix of languages?
     * Doesn't this assume that all languages split sentences alike? */
    public String[] getAllSentences(String text) {
        
    // This function doesn't work if the sentenceDetector object wasn't set up
    if(sentenceDetector == null) return null;

    String[] sentences = sentenceDetector.sentDetect(text);
    return sentences;
    }
 
    public ArrayList<SentenceInfo> getAllSentencesInfo(String[] sentences) {

    if(sentences == null) {
        return null;        
    }

    ArrayList<SentenceInfo> sentencesList = new ArrayList<SentenceInfo>();
    for(int i = 0; i < sentences.length; i++) {
        String sentence = sentences[i];     
        
        //System.err.println(sentence);

        //Language bestLanguage = myCategorizer.predictLanguage(sentence);
        //double confidence = bestLanguage.getConfidence();
        //sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
        
        Language languages[] = myCategorizer.predictLanguages(sentence);
        // languages array already sorted in order of descending confidence
        LanguageInfo[] languagesInfo = new LanguageInfo[NUM_TOP_LANGUAGES];
        for(int j = 0; j < languages.length && j < NUM_TOP_LANGUAGES; j++) {
        String langCode = languages[j].getLang();
        double confidence = languages[j].getConfidence();
        languagesInfo[j] = new LanguageInfo(confidence, langCode);
        }
        
        sentencesList.add(new SentenceInfo(sentence, languagesInfo));
        
    }

    return sentencesList;
    }

    public ArrayList<SentenceInfo> getAllOverlappingSentencesInfo(String[] sentences) {
    
    if(sentences == null) {
        return null;        
    }

    ArrayList<SentenceInfo> sentencesList = new ArrayList<SentenceInfo>();
    for(int i = 1; i < sentences.length; i++) {
        // glue every two adjacent sentences together
        String doubleSentence = sentences[i-1];
        
        String separator = ". ";
        // if the sentence already ends with a terminating punctuation character,
        // then separator is just a space
        doubleSentence = doubleSentence.trim();
        if(doubleSentence.endsWith(".") || doubleSentence.endsWith("?") || doubleSentence.endsWith("!")) {
        separator = " ";
        }
        doubleSentence = doubleSentence + separator + sentences[i]; 
        
        //System.err.println(sentence);

        //Language bestLanguage = myCategorizer.predictLanguage(doubleSentence);
        //double confidence = bestLanguage.getConfidence();
        //sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), doubleSentence));
        
        Language languages[] = myCategorizer.predictLanguages(doubleSentence);
        // languages array already sorted in order of descending confidence
        LanguageInfo[] languagesInfo = new LanguageInfo[NUM_TOP_LANGUAGES];
        
        for(int j = 0; j < languages.length && j < NUM_TOP_LANGUAGES; j++) {
        String langCode = languages[j].getLang();
        double confidence = languages[j].getConfidence();
        languagesInfo[j] = new LanguageInfo(confidence, langCode);
        }
        sentencesList.add(new SentenceInfo(doubleSentence, languagesInfo));
        
    }

    return sentencesList;
    }
    
    /** 
     * In this class' constructor, need to have set up the Sentence Detection Model
     * for the langCode passed in to this function in order for the output to make
     * sense for that language.
     * Function that takes a text and returns those sentences in the requested language.
     * @param text: the string of text from which sentences in the requested
     * language are to be identified and returned.
     * @param langCode: 3 letter code of requested language
     * @param confidenceCutoff: minimum confidence for a SINGLE sentence to be selected
     * even if the language detector determined the requested language as the primary one
     * for that sentence. The confidence cutoff provides an additional check.
     * @return null if no Sentence Detection Model set up in constructor
     * else returns an ArrayList where:
     *   - the first element is the total number of sentences in the text parameter
     *   - remaining elements are the sentences in the text parameter that were in the
     *     requested language.
     */
    public ArrayList<String> getAllSentencesInLanguage(String langCode, String text, double confidenceCutoff)
    {
    // big assumption here: that we can split incoming text into sentences
    // for any language using the sentence model trained for a given language (that of
    // langCode), despite not knowing what language each sentence in the text param are in.
    // Hinges on sentence detection in langCode being similar to all others?

    
    // This function doesn't work if the sentenceDetector object wasn't set up
    if(sentenceDetector == null) return null;
    
    // we'll be storing just those sentences in text that are in the denoted language code
    ArrayList<String> sentencesInLang = new ArrayList<String>();
    // OpenNLP language detection works best with a minimum of 2 sentences
    // See https://opennlp.apache.org/news/model-langdetect-183.html
    // "It is important to note that this model is trained for and works well with
    // longer texts that have at least 2 sentences or more from the same language."
    
    // For evaluating single languages, I used a very small data set and found that
    // if the primary language detected is MRI AND if the confidence is >= 0.1, the
    // results appear reasonably to be in te reo Māori.
    
    String[] sentences = sentenceDetector.sentDetect(text);
    if(sentences == null) {
        sentencesInLang.add("0"); // to indicate 0 sentences in requested language
        return sentencesInLang;
    }

    // add in first element: how many sentences there were in text.
    sentencesInLang.add(Integer.toString(sentences.length));
    
    for(int i = 0; i < sentences.length; i++) {
        String sentence = sentences[i];     
        
        //System.err.println(sentence);

        Language bestLanguage = myCategorizer.predictLanguage(sentence);
        double confidence = bestLanguage.getConfidence();
        
        if(bestLanguage.getLang().equals(langCode) && confidence >= confidenceCutoff) {
        //System.err.println("Adding sentence: " + sentence + "\n");
        sentencesInLang.add(sentence);      
        } //else {
        //System.err.println("SKIPPING sentence: " + sentence + "\n");
        //}
    }
    return sentencesInLang;
    }


    /** @param langCode is 3 letter language code, ISO 639-2/3 
     * https://www.loc.gov/standards/iso639-2/php/code_list.php
     * https://en.wikipedia.org/wiki/ISO_639-3
     * @return true if the input text is Maori (mri) with MINIMUM_CONFIDENCE levels of confidence (if set,
     * else DEFAULT_MINIMUM_CONFIDENCE levels of confidence).
     */
    public boolean isTextInLanguage(String langCode, String text) {
    // Get the most probable language
    Language bestLanguage = myCategorizer.predictLanguage(text);
    doPrint("Best language: " + bestLanguage.getLang());
    doPrint("Best language confidence: " + bestLanguage.getConfidence());

    return (bestLanguage.getLang().equals(langCode) && bestLanguage.getConfidence() >= this.MINIMUM_CONFIDENCE);
    }
    
    
    /**
     * Handle "smaller" textfiles/streams of text read in.
     * Return value is the same as for isTextInLanguage(String langCode, String text);
     */
    public boolean isTextInLanguage(String langCode, BufferedReader reader) throws Exception {
    // https://stackoverflow.com/questions/326390/how-do-i-create-a-java-string-from-the-contents-of-a-file
    
    StringBuilder text = new StringBuilder();
    String line = null;

    
    while((line = reader.readLine()) != null) { // readLine removes newline separator
        text.append(line + "\n"); // add back (unix style) line ending
    }

    return isTextInLanguage(langCode, text.toString());
    }
    

    /**
     * Rudimentary attempt to deal with very large files.
     * Return value is the same as for isTextInLanguage(String langCode, String text);
     */    
    public boolean isLargeTextInLanguage(String langCode, BufferedReader reader) throws Exception {
    // https://stackoverflow.com/questions/326390/how-do-i-create-a-java-string-from-the-contents-of-a-file
    
    final int NUM_LINES = 100; // arbitrary 100 lines read, predict language, calculate confidence

    StringBuilder text = new StringBuilder();
    String line = null;
    
    double cumulativeConfidence = 0;
    int numLoops = 0;
    
    int i = 0;
    String language = null;
    
    while((line = reader.readLine()) != null) { // readLine removes newline separator
        text.append(line + "\n"); // add back (unix style) line ending
        
        i++; // read nth line of numLoop
        
        
        if(i == NUM_LINES) { // arbitrary 100 lines read, predict language, calculate confidence
        
        Language bestLanguage = myCategorizer.predictLanguage(text.toString());
        if(language != null && !bestLanguage.getLang().equals(language)) { // predicted lang of current n lines not the same as predicted lang for prev n lines
            doPrintErr("**** WARNING: text seems to contain content in multiple languages or unable to consistently predict the same language.");           
        }
        language = bestLanguage.getLang();
        cumulativeConfidence += bestLanguage.getConfidence();
        
        doPrintErr("Best predicted language for last " + NUM_LINES + " lines: " + language + "(confidence: " + bestLanguage.getConfidence() + ")"); 
        
        // finished analysing language of NUM_LINES of text
        text = new StringBuilder();
        i = 0;
        numLoops++;
        }       
    }
    
    // process any (remaining) text that was less than n NUM_LINES
    if(!text.toString().equals("")) {
        text.append(line + "\n"); // add back (unix style) line ending      
        i++;
        
        Language bestLanguage = myCategorizer.predictLanguage(text.toString());
        
        if(language != null && !bestLanguage.getLang().equals(language)) { // predicted lang of current n lines not the same as predicted lang for prev n lines
        doPrintErr("**** WARNING: text seems to contain content in multiple languages or unable to consistently predict the same language.");           
        }
        language = bestLanguage.getLang();
        cumulativeConfidence += bestLanguage.getConfidence();
        doPrintErr("Best predicted language for final " + NUM_LINES + " lines: " + language + "(confidence: " + bestLanguage.getConfidence() + ")");
    }
    
    
    int totalLinesRead = numLoops * NUM_LINES + i; // not used
    double avgConfidence = cumulativeConfidence/(numLoops + 1); // not quite the average as the text processed outside the loop may have fewer lines than NUM_LINES
    
    
    return (language.equals(langCode) && avgConfidence >= this.MINIMUM_CONFIDENCE);
    }
    

    /**
     * Prints to STDOUT the predicted languages of the input text in order of descending confidence.
     * UNUSED.
     */
    public void predictedLanguages(String text) {
    // Get an array with the most probable languages
    
    Language[] languages = myCategorizer.predictLanguages(text);
    
    if(languages == null || languages.length <= 0) {
        doPrintErr("No languages predicted for the input text");
    } else {
        for(int i = 0; i < languages.length; i++) {
        doPrint("Language prediction " + i + ": " + languages[i]);
        }
    }
    
    }

    public void doPrint(String msg) {
    if(!this.silentMode) System.out.println(msg);
    }
    public void doPrintErr(String msg) {
    if(!this.silentMode) System.err.println(msg);
    }
    
}