Changeset 33336 for gs3-extensions

Timestamp:

2019-07-20T22:58:17+12:00 (5 years ago)

Author:

ak19

Message:

Major rewrite to make this class more useful to callers. MaoriDetector.java, soon to be renamed, can now be called with a filepath to a txt file or with the hyphen to indicate it should read text from std input (terminate with Ctrl-D as usual). Then it will predict whether the language of the text is in Maori or not depending on the min confidence cut-off value, which presently defaults to 0.5. Still need to support silent mode and update the README with instructions. Now the language detection model file needed by OpenNLP to do the language detection needs to live at OPENNLP_HOME/models/langdetect-183.bin

Location:

gs3-extensions/maori-lang-detection/src

Files:

• MaoriDetector.class (modified) ( previous)
• MaoriDetector.java (modified) (2 diffs)

gs3-extensions/maori-lang-detection/src/MaoriDetector.java

@@ -15 +15 @@
 
 /**
- * Run as:
- *    wharariki:[115]/Scratch/ak19/openNLP-lang-detect/src>javac -cp ".:$OPENNLP_HOME/lib/*" MaoriDetector.java
- *    wharariki:[116]/Scratch/ak19/openNLP-lang-detect/src>java -cp ".:$OPENNLP_HOME/lib/*" MaoriDetector
+ * EXPORT OPENNLP_HOME environment variable to be your apache OpenNLP installation.
+ * Then, to compile this program:
+ *    maori-lang-detection/src$ javac -cp ".:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" MaoriDetector.java
+ * To run this program, one of:
+ *
+ *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/*" MaoriDetector --help
+ *
+ *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/*" MaoriDetector --file <full/path/to/textfile>
+ *
+ *    maori-lang-detection/src$ java -cp ".:$OPENNLP_HOME/lib/*" MaoriDetector -
+ *       which expects text to stream in from standard input.
+ *       If entering text manually, then remember to press Ctrl-D to indicate the usual end of StdIn.
  *
  * https://stackoverflow.com/questions/219585/including-all-the-jars-in-a-directory-within-the-java-classpath
@@ -23 +32 @@
  */
 public class MaoriDetector {
-
+    /** The 3 letter language code for Maori in ISO 639-2 or ISO 639-3 */ 
+    public static final String MAORI_3LETTER_CODE = "mri";
+    public static final double DEFAULT_MINIMUM_CONFIDENCE = 0.50;
+
+    /** Configurable: cut off minimum confidence value,
+    greater or equal to which determines that the best predicted language is acceptable to user of MaoriDetector. */
+    public final double MINIMUM_CONFIDENCE;
+    /** silentMode set to false means MaoriDetector won't print helpful messages while running. Set to true to run silently. */
+    public final boolean silentMode;
+
+    /** Language Detection Model file for OpenNLP is expected to be at $OPENNLP_HOME/models/langdetect-183.bin */
+    private final String LANG_DETECT_MODEL_RELATIVE_PATH = "models" + File.separator + "langdetect-183.bin";
+    private LanguageDetector myCategorizer = null;
+    
     /** 
-     * Taken from our university website
+     * String taken from our university website
      * https://www.waikato.ac.nz/maori/
      */
-    public static final String TEST_INPUT_TEXT = "Ko tēnei te Whare Wānanga o Waikato e whakatau nei i ngā iwi o te ao, ki roto i te riu o te awa e rere nei, ki runga i te whenua e hora nei, ki raro i te taumaru o ngā maunga whakaruru e tau awhi nei.";
-    
-    public static void main(String args[]) {
+    public static final String TEST_MRI_INPUT_TEXT = "Ko tēnei te Whare Wānanga o Waikato e whakatau nei i ngā iwi o te ao, ki roto i te riu o te awa e rere nei, ki runga i te whenua e hora nei, ki raro i te taumaru o ngā maunga whakaruru e tau awhi nei.";
+
+    /** test input string for a negative result */
+    public static final String TEST_ENG_INPUT_TEXT = "The main program exits with -1 if an Exception occurred when attempting to detect the text's language";
+    
+    
+    public MaoriDetector(boolean silentMode) throws Exception {
+    this(silentMode, DEFAULT_MINIMUM_CONFIDENCE);
+    }
+    
+    public MaoriDetector(boolean silentMode, double min_confidence) throws Exception {
+    this.silentMode = silentMode;
+    this.MINIMUM_CONFIDENCE = min_confidence;
+
+    // 1. Check we can find the Language Detect Model file in the correct location (check that $OPENNLP_HOME/models/langdetect-183.bin exists);
+    String langDetectModelPath = System.getenv("OPENNLP_HOME");
+    if(System.getenv("OPENNLP_HOME") == null) {
+        throw new Exception("\n\t*** Environment variable OPENNLP_HOME must be set to your Apache OpenNLP installation folder.");
+    }   
+    langDetectModelPath = langDetectModelPath + File.separator + LANG_DETECT_MODEL_RELATIVE_PATH;
+    File langDetectModelBinFile = new File(langDetectModelPath);
+    if(!langDetectModelBinFile.exists()) {
+        throw new Exception("\n\t*** " + langDetectModelBinFile.getPath() + " doesn't exist."
+                + "\n\t*** Ensure the $OPENNLP_HOME folder contains a 'models' folder with the model file 'langdetect-183.bin' in it.");
+    }
+
+
+    // 2. Set up our language detector Model and the Categorizer for language predictions based on the Model.
     // http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#intro.api
     // https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
-    try (InputStream modelIn = new FileInputStream("/Scratch/ak19/openNLP-lang-detect/langdetect-183.bin")) {
+    try (InputStream modelIn = new FileInputStream(langDetectModelPath)) {
 
         LanguageDetectorModel model = new LanguageDetectorModel(modelIn);
 
         // http://opennlp.apache.org/docs/1.9.1/manual/opennlp.html#tools.langdetect
-        LanguageDetector myCategorizer = new LanguageDetectorME(model);
-
-        // Get the most probable language
-        Language bestLanguage = myCategorizer.predictLanguage(TEST_INPUT_TEXT);
-        System.out.println("Best language: " + bestLanguage.getLang());
-        System.out.println("Best language confidence: " + bestLanguage.getConfidence());
-
-        
-        // Get an array with the most probable languages
-        Language[] languages = myCategorizer.predictLanguages(TEST_INPUT_TEXT);
-        /*
-        if(languages == null || languages.length <= 0) {
-        System.err.println("No languages predicted for the input text");
+        this.myCategorizer = new LanguageDetectorME(model);
+    }/*catch(Exception e) {
+        e.printStackTrace();
+        }*/
+    
+    // instantiating function should handle critical exceptions. Constructors shouldn't.
+    }    
+
+    /**
+     * @return true if the input text is Maori (mri) with MINIMUM_CONFIDENCE levels of confidence (if set,
+     * else DEFAULT_MINIMUM_CONFIDENCE levels of confidence).
+     */
+    public boolean isTextInMaori(String text) { 
+    return isTextInLanguage(MAORI_3LETTER_CODE, text);
+    }
+
+    /** @param langCode is 3 letter language code, ISO 639-2/3 
+     * https://www.loc.gov/standards/iso639-2/php/code_list.php
+     * https://en.wikipedia.org/wiki/ISO_639-3
+     * @return true if the input text is Maori (mri) with MINIMUM_CONFIDENCE levels of confidence (if set,
+     * else DEFAULT_MINIMUM_CONFIDENCE levels of confidence).
+     */
+    public boolean isTextInLanguage(String langCode, String text) {
+    // Get the most probable language
+    Language bestLanguage = myCategorizer.predictLanguage(text);
+    System.out.println("Best language: " + bestLanguage.getLang());
+    System.out.println("Best language confidence: " + bestLanguage.getConfidence());
+
+    return (bestLanguage.getLang().equals(langCode) && bestLanguage.getConfidence() >= this.MINIMUM_CONFIDENCE);
+    }
+    
+    
+    /**
+     * Handle "smaller" textfiles/streams of text read in.
+     * Return value is the same as for isTextInMaori(String text);
+     */
+    public boolean isTextInMaori(BufferedReader reader) throws Exception {
+    return isTextInLanguage(MAORI_3LETTER_CODE, reader);
+    }
+    /**
+     * Handle "smaller" textfiles/streams of text read in.
+     * Return value is the same as for isTextInLanguage(String langCode, String text);
+     */
+    public boolean isTextInLanguage(String langCode, BufferedReader reader) throws Exception {
+    // https://stackoverflow.com/questions/326390/how-do-i-create-a-java-string-from-the-contents-of-a-file
+    
+    StringBuilder text = new StringBuilder();
+    String line = null;
+
+    
+    while((line = reader.readLine()) != null) { // readLine removes newline separator
+        text.append(line + "\n"); // add back (unix style) line ending
+    }
+    return isTextInLanguage(langCode, text.toString());
+    }
+    
+    /*
+     * Need better handling of "larger" textfiles/streams of text read in:
+     * what if multiple languages with high confidence every NUM_LINES read in?
+     * Does this mean the file is multi-lingual with each section dominated by a different language?
+     * How best to convey such information to the user?
+     */
+    /**
+     * Rudimentary attempt to deal with very large files.
+     * Return value is the same as for isTextInMaori(String text);
+     */
+    public boolean isLargeTextInMaori(BufferedReader reader) throws Exception {
+    return isLargeTextInLanguage(MAORI_3LETTER_CODE, reader);
+    }
+
+    /**
+     * Rudimentary attempt to deal with very large files.
+     * Return value is the same as for isTextInLanguage(String langCode, String text);
+     */    
+    public boolean isLargeTextInLanguage(String langCode, BufferedReader reader) throws Exception {
+    // https://stackoverflow.com/questions/326390/how-do-i-create-a-java-string-from-the-contents-of-a-file
+    
+    final int NUM_LINES = 100; // arbitrary 100 lines read, predict language, calculate confidence
+
+    StringBuilder text = new StringBuilder();
+    String line = null;
+    
+    double cumulativeConfidence = 0;
+    int numLoops = 0;
+    
+    int i = 0;
+    String language = null;
+    
+    while((line = reader.readLine()) != null) { // readLine removes newline separator
+        text.append(line + "\n"); // add back (unix style) line ending
+        
+        i++; // read nth line of numLoop
+        
+        
+        if(i == NUM_LINES) { // arbitrary 100 lines read, predict language, calculate confidence
+        
+        
+        Language bestLanguage = myCategorizer.predictLanguage(text.toString());
+        if(language != null && !bestLanguage.getLang().equals(language)) { // predicted lang of current n lines not the same as predicted lang for prev n lines
+            System.err.println("**** WARNING: text seems to contain content in multiple languages or unable to consistently predict the same language.");           
+        }
+        language = bestLanguage.getLang();
+        cumulativeConfidence += bestLanguage.getConfidence();
+        
+        System.err.println("Best predicted language for last " + NUM_LINES + " lines: " + language + "(confidence: " + bestLanguage.getConfidence() + ")"); 
+        
+        // finished analysing language of NUM_LINES of text
+        text = new StringBuilder();
+        i = 0;
+        numLoops++;
+        }       
+    }
+    
+    // process any (remaining) text that was less than n NUM_LINES
+    if(!text.toString().equals("")) {
+        text.append(line + "\n"); // add back (unix style) line ending      
+        i++;
+        
+        Language bestLanguage = myCategorizer.predictLanguage(text.toString());
+        
+        if(language != null && !bestLanguage.getLang().equals(language)) { // predicted lang of current n lines not the same as predicted lang for prev n lines
+        System.err.println("**** WARNING: text seems to contain content in multiple languages or unable to consistently predict the same language.");           
+        }
+        language = bestLanguage.getLang();
+        cumulativeConfidence += bestLanguage.getConfidence();
+        System.err.println("Best predicted language for final " + NUM_LINES + " lines: " + language + "(confidence: " + bestLanguage.getConfidence() + ")");
+    }
+    
+    
+    int totalLinesRead = numLoops * NUM_LINES + i; // not used
+    double avgConfidence = cumulativeConfidence/(numLoops + 1); // not quite the average as the text processed outside the loop may have fewer lines than NUM_LINES
+    
+    
+    return (language.equals(langCode) && avgConfidence >= this.MINIMUM_CONFIDENCE);
+    }
+
+    
+
+    /**
+     * Prints to STDOUT the predicted languages of the input text in order of descending confidence.
+     * Unused.
+     */
+    public void predictedLanguages(String text) {
+    // Get an array with the most probable languages
+    
+    Language[] languages = myCategorizer.predictLanguages(text);
+    
+    if(languages == null || languages.length <= 0) {
+        System.err.println("No languages predicted for the input text");
+    } else {
+        for(int i = 0; i < languages.length; i++) {
+        System.out.println("Language prediction " + i + ": " + languages[i]);
+        }
+    }
+    
+    }
+
+    public static void printUsage() {
+    System.err.println("Run this program with:");
+    System.err.println("\t--help (-h)\tfor seeing this usage message again");
+    System.err.println("\t-\tto have input text read from STDIN (as always, hit Ctrl-D to mark end of text stream)");
+    System.err.println("\t--file (-f)\tto provide an input file path");
+    System.err.println("\t--silent (-s): optional, to run silently and just exit with exit value. [not yet implemented]");
+    System.err.println("\t--min-confidence (-c): optional, to override the default minimum confidence value (" + DEFAULT_MINIMUM_CONFIDENCE + ")");
+    System.err.println("\t\tof the predicted language that will be considered acceptable.");
+    System.err.println();
+    System.err.println("This program terminates with exit value:");
+    System.err.println("\t0 if the input text is in Maori");
+    System.err.println("\t1 if input text is not in Maori");
+    System.err.println();
+    System.err.println("\t-1 if the input arguments were wrong");
+    System.err.println("\t255(!) if an Exception occurred in instantiating the MaoriDetector when attempting to detect the text's language");
+    System.err.println("\t2 if the user asked to run this program with --help/-h.");
+    System.err.println();
+    }
+
+    /**
+     * The main program exits with:
+     *    0 if text is in Maori;
+     *    1 if text is not in Maori;
+     *
+     *    -1 if the input arguments were wrong
+     *    255(!) if an Exception occurred in instantiating the MaoriDetector when attempting to detect the text's language
+     * QTODO: why does the program exit value end up as 255 and not -1 when returnVal remains at -1 on Exception?
+     *    2 if the user asked to run this program with --help/-h.
+     */
+    public static void main(String args[]) {
+    int returnVal = -1;
+
+    // 1. Check input arguments
+    boolean printUsage = false;
+    boolean readFromStdIn = false;
+    File inFile = null;
+    boolean runSilent = false;
+    double minConfidence = -1;
+
+    for (int i = 0; !printUsage && i < args.length; i++) {
+        
+        // check for help first and quit after printing usage
+        if(args[i].equals("--help") || args[i].equals("-h")) {
+        printUsage = true;
+        returnVal = 2;      
+        } else if(args[i].equals("--silent") || args[i].equals("-s")) {
+        runSilent = true;
+        } else if(args[i].equals("--min-confidence") || args[i].equals("-c")) {
+        i++;
+        if(i >= args.length) {
+            System.err.println("ERROR: No minimum confidence value provided with --min-confidence|-c flag.\n");
+            printUsage = true;
+            returnVal = -1;
+        } else {
+            try {
+            minConfidence = Double.parseDouble(args[i]);
+            if(minConfidence < 0 || minConfidence > 1) {
+                throw new NumberFormatException("Number out of range, must be between 0-1");
+            }
+            } catch(NumberFormatException nfe) {
+            System.err.println("ERROR: value for min-confidence is the wrong format or out of range. It must be a (decimal point) number between 0-1.\n");
+            printUsage = true;
+            returnVal = -1;
+            }
+        }
+        } else if(args[i].equals("-")) {
+        readFromStdIn = true;
+        //break; // don't bother continuing to check input arguments for any --file flag if we're told to read from stdin
+        } else if(args[i].equals("--file") || args[i].equals("-f")) {
+        i++;
+        if(i >= args.length) {
+            System.err.println("ERROR: No input file provided with --file|-f flag.\n");
+            printUsage = true;
+            returnVal = -1;
+        } else {
+            String filePath = args[i];
+            inFile = new File(filePath);
+            if(!inFile.isFile()) {
+            System.err.println("ERROR: Can't read text. Input file argument provided does not exist or is not a file.\n");
+            printUsage = true;
+            returnVal = -1;
+            }
+        }
+        } else { // unrecognised input argument
+        System.err.println("ERROR: Unrecognised " + i + "th argument to this program.\n");
+        printUsage = true;
+        returnVal = -1;
+        }
+    }
+
+    if(!readFromStdIn && inFile == null) { // at least one input source must be provided
+        System.err.println("ERROR: must specify source to read text from, either STDIN (-) or input file (--file <file>).\n");
+        printUsage = true;
+        returnVal = -1;
+    }
+    
+    if(readFromStdIn && inFile != null) { // this program can't be asked to read from stdin and from an input file
+        System.err.println("ERROR: instructed to read from both STDIN and from an input file. Not possible.\n");
+        printUsage = true;
+        returnVal = -1;
+    }
+    
+    if(printUsage) {
+        if(!runSilent || returnVal == 2) { // if expressly asked for help or not running silent
+        printUsage();
+        }
+        System.exit(returnVal);
+    }   
+    
+    try {
+        MaoriDetector maoriTextDetector = null;
+        if(minConfidence == -1) {
+        maoriTextDetector = new MaoriDetector(runSilent);
         } else {
-        for(int i = 0; i < languages.length; i++) {
-            System.out.println("Language prediction " + i + ": " + languages[i]);
-        }
-        }*/
+        maoriTextDetector = new MaoriDetector(runSilent, minConfidence);
+        }
+        
+        //boolean textIsInMaori = maoriTextDetector.isTextInMaori(TEST_MRI_INPUT_TEXT);
+        boolean textIsInMaori = false;
+        
+        // Using try with resources, https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
+        if(inFile != null) {
+        System.err.println("Reading text from file " + inFile.getPath());
+        try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
+            textIsInMaori = maoriTextDetector.isTextInMaori(reader);
+        } // let outer try deal with any file/reading exceptions
+        }
+        else if (readFromStdIn) {
+        System.err.println("Waiting to read text from STDIN... (press Ctrl-D when done entering text)>");
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
+            textIsInMaori = maoriTextDetector.isTextInMaori(reader);            
+        } // let outer try deal with any file/reading exceptions
+        }
+    
+        if(textIsInMaori) {
+        returnVal = 0;
+        } else {
+        returnVal = 1;
+        }
         
     } catch(Exception e) {
         e.printStackTrace();
-    }
-    
-    System.err.println("Exitting program...\n");
-    System.exit(0);
-    }
+        
+    } finally {
+        System.err.println("Exitting program with returnVal " + returnVal + "...\n");
+        System.exit(returnVal);
+    }
+    }
+
+    // test hardcoded string
+    public static void oldMain(String args[]) {
+    int returnVal = -1;
+    boolean silentMode = false;
+    
+    try {
+        MaoriDetector maoriTextDetector = new MaoriDetector(silentMode);
+        
+        boolean textIsInMaori = maoriTextDetector.isTextInMaori(TEST_MRI_INPUT_TEXT);
+        if(textIsInMaori) {
+        returnVal = 0;
+        } else {
+        returnVal = 1;
+        }
+        
+    } catch(Exception e) {
+        e.printStackTrace();
+    } finally {
+        System.err.println("Exitting program with returnVal " + returnVal + "...\n");
+        System.exit(returnVal);
+    }
+    }
+
+    
 }