Changeset 33698 for other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

Timestamp:

2019-11-15T23:14:48+13:00 (4 years ago)

Author:

ak19

Message:

Links to more reading

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -81 +81 @@
     /** keep a list to store the text of each page */
     private ArrayList<TextDumpPage> pages;
+
+
+    
+    /** Number of language and confidence results to return for storing in MongoDB
+     * MongoDB runs out of space if storing too many, as we store this info per sentence
+     * and a long text document becomes a very large MongoDB document presumable*/
+    private static final int NUM_TOP_LANGUAGES = 3; // 103 max, in current version of opennlp lang model
+
     
     private boolean isStartOfNewWebPageRecord(String prevLine, String line) {
@@ -232 +240 @@
         int totalSentences = sentences.length;
         int numSentencesInMRI = 0;
-        ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences);
-        ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences);
+        ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences, NUM_TOP_LANGUAGES);
+        ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences, NUM_TOP_LANGUAGES);
 
         WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER/*new ObjectId()*/,
@@ -242 +250 @@
                                      overlappingSentences);
 
+        
         for(SentenceInfo si : singleSentences) {
-            LanguageInfo bestLanguage = si.languagesInfo[0];
-            if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            //LanguageInfo bestLanguage = si.languagesInfo[0];
+            //if(bestLanguage.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            if(si.bestLangCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
             numSentencesInMRI++;
             }
         }
+        
+        
         webpage.setMRISentenceCount(numSentencesInMRI);
         webpage.setContainsMRI((numSentencesInMRI > 0));