Changeset 33651

Timestamp:

2019-11-12T18:11:39+13:00 (4 years ago)

Author:

ak19

Message:

1. Bugfix: overlappingSentences works. 2. storing numSentencesInMaor

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (4 diffs)
• TextLanguageDetector.java (modified) (1 diff)
• WebpageInfo.java (modified) (3 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -180 +180 @@
     https://stackoverflow.com/questions/39433775/mongodb-java-inserting-throws-org-bson-codecs-configuration-codecconfigurationex
     */
+    /**
+     * Inserts a web page into the mongodb. Besides page related metadata and full body text
+     * the language information per sentence and per 2 adjacent sentences also get stored
+     * into the mongodb.
+     */    
     public void insertWebpageInfo(WebpageInfo webpage)
     {
+    int mri_sentence_count = 0;
+    
     // load the webpages db 'table'
     // in mongodb, the equivalent of db tables are called 'collections'
@@ -195 +202 @@
         .append("fetchTime", webpage.fetchTime);
 
-    // DOESN'T WORK, AS EXPECTED, BUT DIDN'T KNOW HOW TO DO IT:
-    //document.put("singleSentences", webpage.singleSentences);
-    //document.put("overlappingSentences", webpage.overlappingSentences);
-
     // INSTEAD, ARRAY OF OBJECTS TO BE INSERTED AS PER:
     // https://stackoverflow.com/questions/15371839/how-to-add-an-array-to-a-mongodb-document-using-java
     List<BasicDBObject> sentencesList = new ArrayList<>();
     for(SentenceInfo sentenceInfo : webpage.singleSentences) {
-        //sentencesList.add(new BasicDBObject("langCode", sentenceInfo.langCode));
-        //sentencesList.add(new BasicDBObject("confidence", sentenceInfo.confidenceLevel));
-        //sentencesList.add(new BasicDBObject("sentence", sentenceInfo.sentence));
+        
         BasicDBObject bsonRecord = new BasicDBObject("langCode", sentenceInfo.langCode);
+        
         bsonRecord.put("confidence", sentenceInfo.confidenceLevel);
         bsonRecord.put("sentence", sentenceInfo.sentence);
         
         sentencesList.add(bsonRecord);
+
+        if(sentenceInfo.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+        mri_sentence_count++;
+        }
+
     }   
     document.put("singleSentences", sentencesList);
@@ -216 +223 @@
     List<BasicDBObject> overlappingSentencesList = new ArrayList<>();
     for(SentenceInfo sentenceInfo : webpage.overlappingSentences) {
+
         BasicDBObject bsonRecord = new BasicDBObject("langCode", sentenceInfo.langCode);
         bsonRecord.put("confidence", sentenceInfo.confidenceLevel);
         bsonRecord.put("sentence", sentenceInfo.sentence);
-        
-        sentencesList.add(bsonRecord);
+        
+        overlappingSentencesList.add(bsonRecord);
     }   
     document.put("overlappingSentences", overlappingSentencesList);
@@ -226 +234 @@
     // also put the full text in there
     document.put("text", webpage.text);
+
+    // also store the count of sentences in MRI
+    webpage.setMRISentenceCount(mri_sentence_count);
+    document.put("mriSentenceCount", mri_sentence_count);
+
     
     collection.insertOne(document);

other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java

@@ -184 +184 @@
     for(int i = 1; i < sentences.length; i++) {
         // glue every two adjacent sentences together
-        String sentence = sentences[i-1];
+        String doubleSentence = sentences[i-1];
         
         String separator = ". ";
         // if the sentence already ends with a terminating punctuation character,
         // then separator is just a space
-        sentence = sentence.trim();
-        if(sentence.endsWith(".") || sentence.endsWith("?") || sentence.endsWith("!")) {
+        doubleSentence = doubleSentence.trim();
+        if(doubleSentence.endsWith(".") || doubleSentence.endsWith("?") || doubleSentence.endsWith("!")) {
         separator = " ";
         }
-        sentence = sentence + separator + sentences[i]; 
+        doubleSentence = doubleSentence + separator + sentences[i]; 
         
         //System.err.println(sentence);
 
-        Language bestLanguage = myCategorizer.predictLanguage(sentence);
+        Language bestLanguage = myCategorizer.predictLanguage(doubleSentence);
         double confidence = bestLanguage.getConfidence();
 
-        sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), sentence));
+        sentencesList.add(new SentenceInfo(confidence, bestLanguage.getLang(), doubleSentence));
     }
 

other-projects/maori-lang-detection/src/org/greenstone/atea/WebpageInfo.java

@@ -5 +5 @@
 public class WebpageInfo {
 
+    private int mriSentenceCount;
+    
     /** db table ids */
     public final long webpageID;
@@ -13 +15 @@
     public final String text;
     public final String URL;
-    public final boolean isMRI;
-    
+    public final boolean isMRI;    
+    
     public final String charEncoding;
     public final String modifiedTime;
@@ -45 +47 @@
     
     }
+
+    public void setMRISentenceCount(int count) {
+    this.mriSentenceCount = count;
+    }
+
+    public int getMRISentenceCount() { return this.mriSentenceCount; }
 }