Changeset 33633 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

Timestamp:

2019-11-08T19:43:39+13:00 (4 years ago)

Author:

ak19

Message:

1. TextLanguageDetector now has methods for collecting all sentences and all overlapping sentences. 2. Renaming NutchTextDumpProcessor.java to NutchTextDumpToCSV.java, since there a new class, NutchTextDumpToMongoDB.java that needs slightly different data structures.

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java (modified) (2 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -47 +47 @@
     private static Logger logger = Logger.getLogger(org.greenstone.atea.MongoDBAccess.class.getName());
     
-    String HOST = "localhost";
-    int PORT = 27017; // mongodb port
-    String PROPS_FILENAME = "config.properties";
-    String DB_NAME = "ateacrawldata";
+    static final String PROPS_FILENAME = "config.properties";
+    public static final String DB_NAME = "anupama"; //"ateacrawldata";
+    public static final String WEBPAGES_COLLECTION = "webpages";
+    public static final String WEBSITES_COLLECTION = "websites";    
     
+    private String HOST = "localhost";
+    private int PORT = 27017; // mongodb port
     private String USERNAME;
-    private String PASSWORD;
-
+    private String PASSWORD;    
 
     private MongoClient mongo = null;
     private MongoDatabase database = null;
+    
     
     public MongoDBAccess() throws Exception {
@@ -118 +120 @@
     logger.info("Credentials: "+ credential);
     }
+
     
+    public void insertWebSiteInfo(int SITE_COUNTER, int siteID, String domainOfSite,
+           int numPages, int numPagesInMRI, int numPagesContainingMRI,
+                  /* TODO: String geoLocationCountryCode, boolean miURL */
+           String siteCrawledTimestamp, String siteCrawlUnfinished, boolean redoCrawl)
+    {
+    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
+    Document document = new Document("id", SITE_COUNTER)
+        .append("siteFolderName", siteID) 
+        .append("domain", domainOfSite) 
+        .append("totalPages", numPages) 
+        .append("numPagesInMRI", numPagesInMRI)
+        .append("numPagesContainingMRI", numPagesContainingMRI)
+        .append("siteCrawledTimestamp", siteCrawledTimestamp)
+        .append("siteCrawlUnfinished", siteCrawlUnfinished)
+        .append("redoCrawl", redoCrawl);
+    collection.insertOne(document); 
+    System.out.println("website info inserted successfully into " + WEBSITES_COLLECTION);  
+    }
 
-    /*
-    public void insertDocument() {
-    MongoCollection<Document> collection = this.database.getCollection("sampleCollection");
+    
+    public void insertWebPage(int WEBPAGE_COUNTER, int site_id, /* id of websites_collection*/
+                  String url, String charEncoding, String modTime, String fetchTime,
+                  boolean isMRI, int totalSentences, int numSentencesInMRI,
+                  ArrayList<SentenceInfo> singleSentences,
+                  ArrayList<SentenceInfo> overlappingSentences)
+    {
+    // load the webpages db 'table'
+    // in mongodb, the equivalent of db tables are called 'collections'
+    MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
+
+    Document document = new Document("id", WEBPAGE_COUNTER)
+        .append("siteid", site_id) 
+        .append("url", url) 
+        .append("charEncoding", charEncoding) 
+        .append("modTime", modTime)
+        .append("fetchTime", fetchTime)
+        .append("isMRI", isMRI)
+        .append("totalSentences", totalSentences)
+        .append("numSentencesInMRI", numSentencesInMRI);
+
+    document.put("singleSentences", singleSentences);
+    document.put("overlappingSentences", overlappingSentences);
+    
+    collection.insertOne(document);
+    System.out.println("website info inserted successfully into " + WEBPAGES_COLLECTION);  
     }
-    */
+    
 
     // TODO: