Changeset 33698

Timestamp:

2019-11-15T23:14:48+13:00 (4 years ago)

Author:

ak19

Message:

Links to more reading

Location:

other-projects/maori-lang-detection

Files:

• MoreReading/mongodb.txt (modified) (2 diffs)
• src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (3 diffs)
• src/org/greenstone/atea/TextLanguageDetector.java (modified) (3 diffs)
• src/org/greenstone/atea/morphia/LanguageInfo.java (modified) (1 diff)
• src/org/greenstone/atea/morphia/SentenceInfo.java (modified) (2 diffs)
• src/org/greenstone/atea/morphia/WebsiteInfo.java (modified) (2 diffs)

other-projects/maori-lang-detection/MoreReading/mongodb.txt

@@ -346 +346 @@
 https://docs.mongodb.com/manual/reference/method/db.collection.find/
 https://docs.mongodb.com/manual/reference/method/db.collection.find/#find-projection
+https://stackoverflow.com/questions/39641925/mongodb-aggregation-framework-to-get-frequencies-of-fields-values
+
+https://exploratory.io/note/kanaugust/0961813761939766
+https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/
+https://docs.mongodb.com/manual/aggregation/
+
 
 Mongo Studio 3T documentation:
@@ -407 +413 @@
 
 
-
+# PROJECTION:
+db.getCollection('Websites').find({geoLocationCountryCode: {$ne:"nz"}}, {geoLocationCountryCode:1, urlContainsLangCodeInpath: 1})
+
+https://docs.mongodb.com/manual/aggregation/
+db.orders.aggregate([
+   { $match: { status: "A" } },
+   { $group: { _id: "$cust_id", total: { $sum: "$amount" } } }
+])
+db.Websites.aggregate({ $match:{urlContainsLangCodeInpath:true}}, $group: {geoLocationCountryCode:1, total: $count})
+
+
+AIMS:
 * Identify where Maori language is online.
 * How can we identify high quality sites that would be good for a corpus.
 (Related work for other languages to quantifiably answer that)
 
+
+
+
+data-preparation
+docs
+

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -81 +81 @@
     /** keep a list to store the text of each page */
     private ArrayList<TextDumpPage> pages;
+
+
+    
+    /** Number of language and confidence results to return for storing in MongoDB
+     * MongoDB runs out of space if storing too many, as we store this info per sentence
+     * and a long text document becomes a very large MongoDB document presumable*/
+    private static final int NUM_TOP_LANGUAGES = 3; // 103 max, in current version of opennlp lang model
+
     
     private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
@@ -232 +240 @@
         int totalSentences = sentences.length;
         int numSentencesInMRI = 0;
-        ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences);
-        ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences);
+        ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences, NUM_TOP_LANGUAGES);
+        ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences, NUM_TOP_LANGUAGES);
 
         WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER/*new ObjectId()*/,
@@ -242 +250 @@
                                      overlappingSentences);
 
+        
         for(SentenceInfo si : singleSentences) {
-            LanguageInfo bestLanguage = si.languagesInfo[0];
-            if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            //LanguageInfo bestLanguage = si.languagesInfo[0];
+            //if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            if(si.bestLangCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
             numSentencesInMRI++;
             }
         }
+        
+        
         webpage.setMRISentenceCount(numSentencesInMRI);
         webpage.setContainsMRI((numSentencesInMRI > 0));

other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java

@@ -53 +53 @@
      */
     public final double MINIMUM_CONFIDENCE;
-
-    /** Number of language and confidence results to return for storing in MongoDB
-     * MongoDB runs out of space if storing too many, as we store this info per sentence
-     * and a long text document becomes a very large MongoDB document presumable*/
-    public final int NUM_TOP_LANGUAGES = 5; // 103 max, in current version of opennlp lang model
     
     /** silentMode set to false means TextLanguageDetector won't print helpful messages while running. Set to true to run silently. */
@@ -162 +157 @@
     }
  
-    public ArrayList<SentenceInfo> getAllSentencesInfo(String[] sentences) {
+    public ArrayList<SentenceInfo> getAllSentencesInfo(String[] sentences, int NUM_TOP_LANGUAGES) {
 
     if(sentences == null) {
@@ -194 +189 @@
     }
 
-    public ArrayList<SentenceInfo> getAllOverlappingSentencesInfo(String[] sentences) {
+    public ArrayList<SentenceInfo> getAllOverlappingSentencesInfo(String[] sentences, int NUM_TOP_LANGUAGES) {
     
     if(sentences == null) {

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/LanguageInfo.java

@@ -5 +5 @@
 @Entity("Languages")
 public class LanguageInfo {
-    public final double confidenceLevel;
+    public final double confidence;
     /** 3 letter lang code */
     public final String langCode;
     public LanguageInfo(double confidence, String langCode) {
-        this.confidenceLevel = confidence;
+        this.confidence = confidence;
         this.langCode = langCode;       
     }

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/SentenceInfo.java

@@ -4 +4 @@
 import java.util.HashMap;
 
+// morphia is used to map Java objects to mongodb
 import dev.morphia.annotations.*;
 
@@ -11 +12 @@
 
     public final String sentence;
-    public final Map<String, Double> languageToConfidenceMap;
-    @Embedded
-    public final LanguageInfo[] languagesInfo; // array of langCode and confidence value pairs
-
+    //public final Map<String, Double> languageToConfidenceMap;
+    public final String bestLangCode;
+    public final double bestLangConfidence;    
+    //@Embedded
+    //public final LanguageInfo[] languagesInfo; // array of langCode and confidence value pairs
+    
     
     public SentenceInfo(String sentence, LanguageInfo[] languages) {
-    this.sentence = sentence;
-    this.languagesInfo = languages;
+    this.sentence = sentence;   
+    if(languages.length >= 1) {
+        this.bestLangCode = languages[0].langCode;
+        this.bestLangConfidence = languages[0].confidence;
+    } else {
+        this.bestLangCode = null;
+        this.bestLangConfidence = 0.0;
+    }
 
+    //this.languagesInfo = null;
+    //this.languagesInfo = languages;
+
+    
     // let's store (langCode -> confidence) lookup in Map:
-    this.languageToConfidenceMap = new HashMap<String, Double>();   
+    //this.languageToConfidenceMap = new HashMap<String, Double>();
+    /*
     for(LanguageInfo li : languages) {
         String langCode = li.langCode;
-        Double confidence = new Double(li.confidenceLevel);
+        Double confidence = new Double(li.confidence);
         languageToConfidenceMap.put(langCode, confidence);
     }
+    */
     }
     
 }
 
-// BACK WHEN WE ONLY STORED THE BEST PREDICTED LANGUAGE META FOR EACH SENTENCE:
-/*
-@Entity("Sentences")
-public class SentenceInfo {
-    public final double confidenceLevel;
-    // 3 letter lang code
-    public final String langCode;
-    public final String sentence;
-    
-    public SentenceInfo(double confidence, String langCode, String sentence) {
-    this.confidenceLevel = confidence;
-    this.langCode = langCode;
-    this.sentence = sentence;
-    }
-}
-*/

other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java

@@ -19 +19 @@
     
     public final String geoLocationCountryCode;
-    public final boolean urlContainsLangCodeInpath;
+    public final boolean urlContainsLangCodeInPath;
     
     public WebsiteInfo(/*int siteCount,*/ String siteFolderName, String domainOfSite,
                int totalPages, int countOfWebPagesWithBodyText, int numPagesInMRI,
                long siteCrawledTimestamp, boolean siteCrawlUnfinished, boolean redoCrawl,
-               String geoLocationCountryCode, boolean urlContainsLangCodeInpath)
+               String geoLocationCountryCode, boolean urlContainsLangCodeInPath)
     {
     //this.id = siteCount;
@@ -39 +39 @@
     
     this.geoLocationCountryCode = geoLocationCountryCode;
-    this.urlContainsLangCodeInpath = urlContainsLangCodeInpath;
+    this.urlContainsLangCodeInPath = urlContainsLangCodeInPath;
     }
 }