Changeset 33674 for other-projects

Timestamp:

2019-11-15T00:21:31+13:00 (4 years ago)

Author:

ak19

Message:

Changes to support the top 5 predicted langcodes and their confidence values per sentence/overlapping sentence (all 103 made some documents, like of site 00006, too big too go into mongodb). Have re-run the NutchTextDumpToMongDB to send the new form of the docs into mongodb.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• NutchTextDumpToMongoDB.java (modified) (3 diffs)
• TextLanguageDetector.java (modified) (5 diffs)
• morphia/LanguageInfo.java (added)
• morphia/SentenceInfo.java (modified) (2 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -243 +243 @@
 
         for(SentenceInfo si : singleSentences) {
-            if(si.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            LanguageInfo bestLanguage = si.languagesInfo[0];
+            if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
             numSentencesInMRI++;
             }
@@ -251 +252 @@
         
         //mongodbAccess.insertWebpageInfo(webpage);
+        // Uses morphia to save to mongodb, see https://www.baeldung.com/mongodb-morphia
         mongodbAccess.datastore.save(webpage);
         }
     }
     }
-
-    /*
-    public void printSiteStats() {
-    
-    
-    logger.info("------------- " + this.siteID + " SITE STATS -----------");
-
-    logger.info("SITE DOMAIN: " + this.domainOfSite);
-    logger.info("Total number of web pages in site: " + pages.size());
-    logger.info("Of these, the number of pages in Māori (mri) were: " + this.pagesInMRI.size());
-    
-    if(pagesInMRI.size() > 0) {
-        logger.info("The following were the pages detected by OpenNLP as being in Māori with " + maoriTxtDetector.MINIMUM_CONFIDENCE + " confidence");
-        for(MRIWebPageStats mriWebPageInfo : pagesInMRI) {
-        logger.info(mriWebPageInfo.toString());
-        }
-    }
-
-    logger.info("                      -----------                   ");
-    if(pagesContainingMRI.size() > 0) {     
-        logger.info("The following pages weren't detected as primarily being in Māori");
-        logger.info("But still contained sentences detected as Māori");
-        for(MRIWebPageStats mriWebPageInfo : pagesContainingMRI) {
-        logger.info(mriWebPageInfo.toString());
-        }
-        
-    } else {
-        logger.info("No further pages detected as containing any sentences in MRI");       
-    }
-    logger.info("                      -----------                   ");
-    }
-    */
-
     
 
@@ -328 +297 @@
 
     //mongodbAccess.insertWebsiteInfo(website);
+    // Uses morphia to save to mongodb, see https://www.baeldung.com/mongodb-morphia
     mongodbAccess.datastore.save(website);
     }

other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java

@@ -29 +29 @@
 import java.util.ArrayList;
 
-import org.greenstone.atea.morphia.*;
+import org.greenstone.atea.morphia.SentenceInfo;
+import org.greenstone.atea.morphia.LanguageInfo;
 
 /**
@@ -52 +53 @@
      */
     public final double MINIMUM_CONFIDENCE;
+
+    /** Number of language and confidence results to return for storing in MongoDB
+     * MongoDB runs out of space if storing too many, as we store this info per sentence
+     * and a long text document becomes a very large MongoDB document presumable*/
+    public final int NUM_TOP_LANGUAGES = 5; // 103 max, in current version of opennlp lang model
     
     /** silentMode set to false means TextLanguageDetector won't print helpful messages while running. Set to true to run silently. */
@@ -168 +174 @@
         //System.err.println(sentence);
 
-        Language bestLanguage = myCategorizer.predictLanguage(sentence);
-        double confidence = bestLanguage.getConfidence();
-
-        sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
+        //Language bestLanguage = myCategorizer.predictLanguage(sentence);
+        //double confidence = bestLanguage.getConfidence();
+        //sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
+        
+        Language languages[] = myCategorizer.predictLanguages(sentence);
+        // languages array already sorted in order of descending confidence
+        LanguageInfo[] languagesInfo = new LanguageInfo[NUM_TOP_LANGUAGES];
+        for(int j = 0; j < languages.length && j < NUM_TOP_LANGUAGES; j++) {
+        String langCode = languages[j].getLang();
+        double confidence = languages[j].getConfidence();
+        languagesInfo[j] = new LanguageInfo(confidence, langCode);
+        }
+        
+        sentencesList.add(new SentenceInfo(sentence, languagesInfo));
+        
     }
 
@@ -199 +216 @@
         //System.err.println(sentence);
 
-        Language bestLanguage = myCategorizer.predictLanguage(doubleSentence);
-        double confidence = bestLanguage.getConfidence();
-
-        sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), doubleSentence));
+        //Language bestLanguage = myCategorizer.predictLanguage(doubleSentence);
+        //double confidence = bestLanguage.getConfidence();
+        //sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), doubleSentence));
+        
+        Language languages[] = myCategorizer.predictLanguages(doubleSentence);
+        // languages array already sorted in order of descending confidence
+        LanguageInfo[] languagesInfo = new LanguageInfo[NUM_TOP_LANGUAGES];
+        
+        for(int j = 0; j < languages.length && j < NUM_TOP_LANGUAGES; j++) {
+        String langCode = languages[j].getLang();
+        double confidence = languages[j].getConfidence();
+        languagesInfo[j] = new LanguageInfo(confidence, langCode);
+        }
+        sentencesList.add(new SentenceInfo(doubleSentence, languagesInfo));
+        
     }
 
@@ -305 +333 @@
         text.append(line + "\n"); // add back (unix style) line ending
     }
+
     return isTextInLanguage(langCode, text.toString());
     }

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/SentenceInfo.java

@@ -1 +1 @@
 package org.greenstone.atea.morphia;
+
+import java.util.Map;
+import java.util.HashMap;
 
 import dev.morphia.annotations.*;
 
+
+@Entity("Sentences")
+public class SentenceInfo {
+
+    public final String sentence;
+    public final Map<String, Double> languageToConfidenceMap;
+    @Embedded
+    public final LanguageInfo[] languagesInfo; // array of langCode and confidence value pairs
+
+    
+    public SentenceInfo(String sentence, LanguageInfo[] languages) {
+    this.sentence = sentence;
+    this.languagesInfo = languages;
+
+    // let's store (langCode -> confidence) lookup in Map:
+    this.languageToConfidenceMap = new HashMap<String, Double>();   
+    for(LanguageInfo li : languages) {
+        String langCode = li.langCode;
+        Double confidence = new Double(li.confidenceLevel);
+        languageToConfidenceMap.put(langCode, confidence);
+    }
+    }
+    
+}
+
+// BACK WHEN WE ONLY STORED THE BEST PREDICTED LANGUAGE META FOR EACH SENTENCE:
+/*
 @Entity("Sentences")
 public class SentenceInfo {
     public final double confidenceLevel;
-    /** 3 letter lang code */
+    // 3 letter lang code
     public final String langCode;
     public final String sentence;
@@ -16 +46 @@
     }
 }
+*/