Changeset 33652

Timestamp:

2019-11-12T20:41:13+13:00 (4 years ago)

Author:

ak19

Message:

Introducing morphia subpackage

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (3 diffs)
• NutchTextDumpToMongoDB.java (modified) (4 diffs)
• TextDumpPage.java (modified) (3 diffs)
• TextLanguageDetector.java (modified) (1 diff)
• morphia (added)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -24 +24 @@
 import org.apache.log4j.Logger;
 
+import org.greenstone.atea.morphia.*;
+import dev.morphia.*;
 
 /** 
@@ -68 +70 @@
     private MongoClient mongo = null;
     private MongoDatabase database = null;
-    
-    
+
+    /** 
+     * Mongodb Client handle via morphia, which handles the ODM (object document mapper)
+     * for MongoDB
+    */
+    public Datastore datastore = null;
+    
     public MongoDBAccess() throws Exception {
     boolean success = false;
@@ -141 +148 @@
     this.database = mongo.getDatabase(DB_NAME); 
     */
+
+    Morphia morphia = new Morphia();
+    morphia.mapPackage("com.greenstone.atea.morphia");
+    datastore = morphia.createDatastore(mongo, DB_NAME);
+    datastore.ensureIndexes();
     
     }

other-projects/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpToMongoDB.java

@@ -9 +9 @@
 import org.apache.commons.csv.*;
 import org.apache.log4j.Logger;
+
+//import org.bson.types.ObjectId;
+    
+import org.greenstone.atea.morphia.*;
 
 
@@ -223 +227 @@
         String[] sentences = maoriTxtDetector.getAllSentences(text);
         int totalSentences = sentences.length;
+        int numSentencesInMRI = 0;
         ArrayList<SentenceInfo> singleSentences = maoriTxtDetector.getAllSentencesInfo(sentences);
         ArrayList<SentenceInfo> overlappingSentences = maoriTxtDetector.getAllOverlappingSentencesInfo(sentences);
-        
-        WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER,
-                                     SITE_COUNTER,
+
+        WebpageInfo webpage = page.convertStoredDataToWebpageInfo(WEBPAGE_COUNTER/*new ObjectId()*/,
+                                     this.siteID/*SITE_COUNTER*/,
                                      isMRI,
                                      totalSentences,
@@ -233 +238 @@
                                      overlappingSentences);
 
-        
-        mongodbAccess.insertWebpageInfo(webpage);
+        for(SentenceInfo si : singleSentences) {
+            if(si.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
+            numSentencesInMRI++;
+            }
+        }
+        webpage.setMRISentenceCount(numSentencesInMRI);
+        webpage.setContainsMRI((numSentencesInMRI > 0));
+        
+        //mongodbAccess.insertWebpageInfo(webpage);
+        mongodbAccess.datastore.save(webpage);
         }
     }
@@ -291 +304 @@
     }
     
-    //File geoLiteCityDatFile = new File(this.getClass().getResource("GeoLiteCity.dat").getFile());
-    //this.geoLocationCountryCode = getCountryCodeOfDomain(this.domainOfSite, geoLiteCityDatFile);
+    File geoLiteCityDatFile = new File(this.getClass().getClassLoader().getResource("GeoLiteCity.dat").getFile());
+    try {
+        this.geoLocationCountryCode = Utility.getCountryCodeOfDomain(this.domainOfSite, geoLiteCityDatFile);
+    } catch(Exception e) {
+        e.printStackTrace();
+        this.geoLocationCountryCode = null;
+    }
 
     int totalPages = pages.size();  
 
-    WebsiteInfo website = new WebsiteInfo(SITE_COUNTER, this.siteID, this.domainOfSite,
+    WebsiteInfo website = new WebsiteInfo(/*SITE_COUNTER,*/ this.siteID, this.domainOfSite,
           totalPages, this.countOfWebPagesWithBodyText, this.numPagesInMRI, 
           this.siteCrawledTimestamp, this.siteCrawlUnfinished, redoCrawl,
           this.geoLocationCountryCode, this.urlContainsLangCodeInPath);
 
-    mongodbAccess.insertWebsiteInfo(website);
-    
+    //mongodbAccess.insertWebsiteInfo(website);
+    mongodbAccess.datastore.save(website);
     }
 

other-projects/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

@@ -8 +8 @@
 import org.apache.log4j.Logger;
 
+import org.greenstone.atea.morphia.*;
 
 public class TextDumpPage {
@@ -178 +179 @@
      */
     public WebpageInfo convertStoredDataToWebpageInfo(
-      long webpageID, int websiteID, boolean isMRI, int totalSentences,
+      long webpageID, String siteID /*int websiteID*/, boolean isMRI, int totalSentences,
       ArrayList<SentenceInfo> singleSentences, ArrayList<SentenceInfo> overlappingSentences)
     {
@@ -188 +189 @@
     String fetchTime = getFetchTime();
 
-    WebpageInfo webpage = new WebpageInfo(webpageID, websiteID,
+    WebpageInfo webpage = new WebpageInfo(webpageID, siteID/*websiteID,*/,
                           pageText, pageURL, isMRI, totalSentences,
                           charEncoding, modifiedTime, fetchTime,

other-projects/maori-lang-detection/src/org/greenstone/atea/TextLanguageDetector.java

@@ -29 +29 @@
 import java.util.ArrayList;
 
+import org.greenstone.atea.morphia.*;
+
 /**
  * EXPORT OPENNLP_HOME environment variable to be your apache OpenNLP installation.