Index: /other-projects/maori-lang-detection/MoreReading/mongodb.txt
===================================================================
--- /other-projects/maori-lang-detection/MoreReading/mongodb.txt	(revision 33697)
+++ /other-projects/maori-lang-detection/MoreReading/mongodb.txt	(revision 33698)
@@ -346,4 +346,10 @@
 https://docs.mongodb.com/manual/reference/method/db.collection.find/
 https://docs.mongodb.com/manual/reference/method/db.collection.find/#find-projection
+https://stackoverflow.com/questions/39641925/mongodb-aggregation-framework-to-get-frequencies-of-fields-values
+
+https://exploratory.io/note/kanaugust/0961813761939766
+https://docs.mongodb.com/manual/tutorial/project-fields-from-query-results/
+https://docs.mongodb.com/manual/aggregation/
+
 
 Mongo Studio 3T documentation:
@@ -407,7 +413,24 @@
 
 
-
+# PROJECTION:
+db.getCollection('Websites').find({geoLocationCountryCode: {$ne:"nz"}}, {geoLocationCountryCode:1, urlContainsLangCodeInpath: 1})
+
+https://docs.mongodb.com/manual/aggregation/
+db.orders.aggregate([
+   { $match: { status: "A" } },
+   { $group: { _id: "$cust_id", total: { $sum: "$amount" } } }
+])
+db.Websites.aggregate({ $match:{urlContainsLangCodeInpath:true}}, $group: {geoLocationCountryCode:1, total: $count})
+
+
+AIMS:
 * Identify where Maori language is online.
 * How can we identify high quality sites that would be good for a corpus.
 (Related work for other languages to quantifiably answer that)
 
+
+
+
+data-preparation
+docs
+
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java	(revision 33697)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java	(revision 33698)
@@ -81,4 +81,12 @@
     /** keep a list to store the text of each page */
     private ArrayList<TextDumpPage> pages;
+
+
+    
+    /** Number of language and confidence results to return for storing in MongoDB
+     * MongoDB runs out of space if storing too many, as we store this info per sentence
+     * and a long text document becomes a very large MongoDB document presumable*/
+    private static final int NUM_TOP_LANGUAGES = 3; // 103 max, in current version of opennlp lang model
+
     
     private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
@@ -232,6 +240,6 @@
 		int totalSentences = sentences.length;
 		int numSentencesInMRI = 0;
-		ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences);
-		ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences);
+		ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences, NUM_TOP_LANGUAGES);
+		ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences, NUM_TOP_LANGUAGES);
 
 		WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER/*new ObjectId()*/,
@@ -242,10 +250,14 @@
 								     overlappingSentences);
 
+		
 		for(SentenceInfo si : singleSentences) {
-		    LanguageInfo bestLanguage = si.languagesInfo[0];
-		    if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+		    //LanguageInfo bestLanguage = si.languagesInfo[0];
+		    //if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+		    if(si.bestLangCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
 			numSentencesInMRI++;
 		    }
 		}
+		
+		
 		webpage.setMRISentenceCount(numSentencesInMRI);
 		webpage.setContainsMRI((numSentencesInMRI > 0));
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java	(revision 33697)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java	(revision 33698)
@@ -53,9 +53,4 @@
      */
     public final double MINIMUM_CONFIDENCE;
-
-    /** Number of language and confidence results to return for storing in MongoDB
-     * MongoDB runs out of space if storing too many, as we store this info per sentence
-     * and a long text document becomes a very large MongoDB document presumable*/
-    public final int NUM_TOP_LANGUAGES = 5; // 103 max, in current version of opennlp lang model
     
     /** silentMode set to false means TextLanguageDetector won't print helpful messages while running. Set to true to run silently. */
@@ -162,5 +157,5 @@
     }
  
-    public ArrayList<SentenceInfo> getAllSentencesInfo(String[] sentences) {
+    public ArrayList<SentenceInfo> getAllSentencesInfo(String[] sentences, int NUM_TOP_LANGUAGES) {
 
 	if(sentences == null) {
@@ -194,5 +189,5 @@
     }
 
-    public ArrayList<SentenceInfo> getAllOverlappingSentencesInfo(String[] sentences) {
+    public ArrayList<SentenceInfo> getAllOverlappingSentencesInfo(String[] sentences, int NUM_TOP_LANGUAGES) {
 	
 	if(sentences == null) {
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/LanguageInfo.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/LanguageInfo.java	(revision 33697)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/LanguageInfo.java	(revision 33698)
@@ -5,9 +5,9 @@
 @Entity("Languages")
 public class LanguageInfo {
-    public final double confidenceLevel;
+    public final double confidence;
     /** 3 letter lang code */
     public final String langCode;
     public LanguageInfo(double confidence, String langCode) {
-	    this.confidenceLevel = confidence;
+	    this.confidence = confidence;
 	    this.langCode = langCode;	    
     }
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/SentenceInfo.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/SentenceInfo.java	(revision 33697)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/SentenceInfo.java	(revision 33698)
@@ -4,4 +4,5 @@
 import java.util.HashMap;
 
+// morphia is used to map Java objects to mongodb
 import dev.morphia.annotations.*;
 
@@ -11,38 +12,36 @@
 
     public final String sentence;
-    public final Map<String, Double> languageToConfidenceMap;
-    @Embedded
-    public final LanguageInfo[] languagesInfo; // array of langCode and confidence value pairs
-
+    //public final Map<String, Double> languageToConfidenceMap;
+    public final String bestLangCode;
+    public final double bestLangConfidence;    
+    //@Embedded
+    //public final LanguageInfo[] languagesInfo; // array of langCode and confidence value pairs
+    
     
     public SentenceInfo(String sentence, LanguageInfo[] languages) {
-	this.sentence = sentence;
-	this.languagesInfo = languages;
+	this.sentence = sentence;	
+	if(languages.length >= 1) {
+	    this.bestLangCode = languages[0].langCode;
+	    this.bestLangConfidence = languages[0].confidence;
+	} else {
+	    this.bestLangCode = null;
+	    this.bestLangConfidence = 0.0;
+	}
 
+	//this.languagesInfo = null;
+	//this.languagesInfo = languages;
+
+	
 	// let's store (langCode -> confidence) lookup in Map:
-	this.languageToConfidenceMap = new HashMap<String, Double>();	
+	//this.languageToConfidenceMap = new HashMap<String, Double>();
+	/*
 	for(LanguageInfo li : languages) {
 	    String langCode = li.langCode;
-	    Double confidence = new Double(li.confidenceLevel);
+	    Double confidence = new Double(li.confidence);
 	    languageToConfidenceMap.put(langCode, confidence);
 	}
+	*/
     }
     
 }
 
-// BACK WHEN WE ONLY STORED THE BEST PREDICTED LANGUAGE META FOR EACH SENTENCE:
-/*
-@Entity("Sentences")
-public class SentenceInfo {
-    public final double confidenceLevel;
-    // 3 letter lang code
-    public final String langCode;
-    public final String sentence;
-    
-    public SentenceInfo(double confidence, String langCode, String sentence) {
-	this.confidenceLevel = confidence;
-	this.langCode = langCode;
-	this.sentence = sentence;
-    }
-}
-*/
Index: /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java
===================================================================
--- /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java	(revision 33697)
+++ /other-projects/maori-lang-detection/src/org/greenstone/atea/morphia/WebsiteInfo.java	(revision 33698)
@@ -19,10 +19,10 @@
     
     public final String geoLocationCountryCode;
-    public final boolean urlContainsLangCodeInpath;
+    public final boolean urlContainsLangCodeInPath;
 	
     public WebsiteInfo(/*int siteCount,*/ String siteFolderName, String domainOfSite,
 		       int totalPages, int countOfWebPagesWithBodyText, int numPagesInMRI,
 		       long siteCrawledTimestamp, boolean siteCrawlUnfinished, boolean redoCrawl,
-		       String geoLocationCountryCode, boolean urlContainsLangCodeInpath)
+		       String geoLocationCountryCode, boolean urlContainsLangCodeInPath)
     {
 	//this.id = siteCount;
@@ -39,5 +39,5 @@
 	
 	this.geoLocationCountryCode = geoLocationCountryCode;
-	this.urlContainsLangCodeInpath = urlContainsLangCodeInpath;
+	this.urlContainsLangCodeInPath = urlContainsLangCodeInPath;
     }
 }
