source: other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java@ 33652

package org.greenstone.atea;

//import org.bson.BSONObject;
    
import com.mongodb.client.MongoCollection; 
import com.mongodb.client.MongoDatabase; 
//import com.mongodb.client.MongoIterable;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient; 
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.MongoClientOptions;

import org.bson.Document;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;


import org.apache.log4j.Logger;

import org.greenstone.atea.morphia.*;
import dev.morphia.*;

/** 
 * https://www.tutorialspoint.com/mongodb/mongodb_java.htm
 *
 * TO COMPILE:
 *    maori-lang-detection/src$
 *       javac -cp ".:../conf:../lib/*" org/greenstone/atea/MongoDBAccess.java
 *
 * TO RUN:
 *       java -cp ".:../conf:../lib/*" org.greenstone.atea.MongoDBAccess
 *
 * Manually connecting to mongodb from client:
 *    mongo 'mongodb://mongodb.cms.waikato.ac.nz:27017' -u USERNAME -p
 * Then after connecting with pwd, type:
 *    use DBNAME
 *
 * Or connect to mongodb and specify db in one statement:
 *    mongo 'mongodb://mongodb.cms.waikato.ac.nz:27017/DBNAME?authSource=admin' -u USERNAME -p
 *
 * Some links:
 *   - https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection
 *   - https://docs.mongodb.com/manual/reference/glossary/ (particularly "collection")
 *   - https://tecadmin.net/tutorial/mongodb/drop-collection/
 * IMPORTANT LINK:
 *   - https://www.mongodb.com/blog/post/6-rules-of-thumb-for-mongodb-schema-design-part-1
 *
 */
public class MongoDBAccess implements AutoCloseable {

    private static Logger logger = Logger.getLogger(org.greenstone.atea.MongoDBAccess.class.getName());
    
    static final String PROPS_FILENAME = "config.properties";
    public static final String WEBPAGES_COLLECTION = "webpages";
    public static final String WEBSITES_COLLECTION = "websites";    

    // configuration details, some with fallback values
    private String HOST = "localhost";
    private int PORT = 27017; // mongodb port
    private String USERNAME;
    private String PASSWORD;    
    private String DB_NAME ="ateacrawldata";
    
    private MongoClient mongo = null;
    private MongoDatabase database = null;

    /** 
     * Mongodb Client handle via morphia, which handles the ODM (object document mapper)
     * for MongoDB
    */
    public Datastore datastore = null;
    
    public MongoDBAccess() throws Exception {
    boolean success = false;
    
    // Read in the username and password from our props file
    Properties props = new Properties();

    //File propsFile = new File(PROPS_FILENAME);
    //logger.debug("*** Conf props filename: " + propsFile.getAbsolutePath());
    try {
        props.load(getClass().getClassLoader().getResourceAsStream(PROPS_FILENAME));        
    } catch(Exception e) {
        logger.error(e);
    }
    
    
    USERNAME = props.getProperty("mongodb.user", "");
    if(USERNAME.equals("")) {
        USERNAME = "root";
        logger.warn("WARNING: No sensible value for mongodb.user specified in " + PROPS_FILENAME + ". Attempting to use: " + USERNAME);
            }
    PASSWORD = props.getProperty("mongodb.pwd");

    logger.debug("Got pwd: " + PASSWORD);
    
    if(PASSWORD != null && PASSWORD.equals("CHANGEME")) {
        
        success = false;
        throw new Exception("************ FATAL ERROR: Change DB password in properties file " + PROPS_FILENAME);       
    }

    HOST = props.getProperty("mongodb.host", HOST);
    String port = props.getProperty("mongodb.port", Integer.toString(PORT));
    PORT = Integer.parseInt(port);
    DB_NAME = props.getProperty("mongodb.dbname", DB_NAME);

    logger.info("Connecting to mongodb with:");
    logger.info(" - host:    " + HOST);
    logger.info(" - port:    " + PORT);
    logger.info(" - user:    " + USERNAME);
    logger.info(" - db name: " + DB_NAME);  
    }

    /** 
     * Since we have only a single MongoClient, don't need to call close/disconnect on it as per
     * https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection
     */
    public void connectToDB() throws Exception {
    
    // Creating a Mongo client 
    mongo = new MongoClient( HOST, PORT ); 
    
    // Creating Credentials 
    MongoCredential credential; 
    credential = MongoCredential.createCredential(USERNAME, DB_NAME, PASSWORD.toCharArray()); 
    System.out.println("Connected to the database successfully");  
    
    // Accessing the database 
    this.database = mongo.getDatabase(DB_NAME); 
    logger.info("Credentials: "+ credential);

    /*
    MongoCredential credential; 
    credential = MongoCredential.createCredential(USERNAME, DB_NAME, PASSWORD.toCharArray());
    logger.info("Credentials: "+ credential);
    
    // Create our Mongo client
    mongo = new MongoClient( new ServerAddress(HOST, PORT), credential, new MongoClientOptions.Builder().build());
    System.out.println("Connected to the database successfully");

    this.database = mongo.getDatabase(DB_NAME); 
    */

    Morphia morphia = new Morphia();
    morphia.mapPackage("com.greenstone.atea.morphia");
    datastore = morphia.createDatastore(mongo, DB_NAME);
    datastore.ensureIndexes();
    
    }

    // TODO: which fields should be indexed?

    public void showCollections() {
    //MongoIterable<String> colls = this.database.listCollectionNames();
    for(String coll : this.database.listCollectionNames()) {
        System.err.println("coll: " + coll);
    }
    }
    
    
    public void insertWebsiteInfo(WebsiteInfo website)
    {
    MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
    Document document = new Document("_id", website.id)
        .append("siteFolderName", website.siteFolderName) 
        .append("domain", website.domain) 
        .append("totalPages", website.totalPages)
        .append("numPagesWithBodyText", website.countOfWebPagesWithBodyText)
        .append("numPagesInMRI", website.numPagesInMRI)
        .append("siteCrawledTimestamp", website.siteCrawledTimestamp)
        .append("siteCrawlUnfinished", website.siteCrawlUnfinished)
        .append("redoCrawl", website.redoCrawl);

    document.put("urlContainsLangCodeInpath", website.urlContainsLangCodeInpath);
    if(website.geoLocationCountryCode != null && !website.geoLocationCountryCode.equals("")) {
        document.put("countryCode", website.geoLocationCountryCode);
    }

    collection.insertOne(document); 
    logger.debug("Website info for " + website.id + "(" + website.siteFolderName + ")"
               + " inserted successfully into " + WEBSITES_COLLECTION);  
    }

    /* TODO:
    https://stackoverflow.com/questions/39433775/mongodb-java-inserting-throws-org-bson-codecs-configuration-codecconfigurationex
    */
    /**
     * Inserts a web page into the mongodb. Besides page related metadata and full body text
     * the language information per sentence and per 2 adjacent sentences also get stored
     * into the mongodb.
     */    
    public void insertWebpageInfo(WebpageInfo webpage)
    {
    int mri_sentence_count = 0;
    
    // load the webpages db 'table'
    // in mongodb, the equivalent of db tables are called 'collections'
    MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);

    Document document = new Document("_id", webpage.webpageID)
        .append("siteid", webpage.websiteID)
        .append("url", webpage.URL)
        .append("isMRI", webpage.isMRI)
        .append("totalSentences", webpage.totalSentences)
        .append("charEncoding", webpage.charEncoding) 
        .append("modTime", webpage.modifiedTime)
        .append("fetchTime", webpage.fetchTime);

    // INSTEAD, ARRAY OF OBJECTS TO BE INSERTED AS PER:
    // https://stackoverflow.com/questions/15371839/how-to-add-an-array-to-a-mongodb-document-using-java
    List<BasicDBObject> sentencesList = new ArrayList<>();
    for(SentenceInfo sentenceInfo : webpage.singleSentences) {
        
        BasicDBObject bsonRecord = new BasicDBObject("langCode", sentenceInfo.langCode);
        
        bsonRecord.put("confidence", sentenceInfo.confidenceLevel);
        bsonRecord.put("sentence", sentenceInfo.sentence);
        
        sentencesList.add(bsonRecord);

        if(sentenceInfo.langCode.equals(MaoriTextDetector.MAORI_3LETTER_CODE)) {
        mri_sentence_count++;
        }

    }   
    document.put("singleSentences", sentencesList);
    
    List<BasicDBObject> overlappingSentencesList = new ArrayList<>();
    for(SentenceInfo sentenceInfo : webpage.overlappingSentences) {

        BasicDBObject bsonRecord = new BasicDBObject("langCode", sentenceInfo.langCode);
        bsonRecord.put("confidence", sentenceInfo.confidenceLevel);
        bsonRecord.put("sentence", sentenceInfo.sentence);
        
        overlappingSentencesList.add(bsonRecord);
    }   
    document.put("overlappingSentences", overlappingSentencesList);
    
    // also put the full text in there
    document.put("text", webpage.text);

    // also store the count of sentences in MRI
    webpage.setMRISentenceCount(mri_sentence_count);
    document.put("mriSentenceCount", mri_sentence_count);

    
    collection.insertOne(document);
    logger.debug("\nwebpage info for " + webpage.webpageID + " inserted successfully into " + WEBPAGES_COLLECTION);  
    }

    /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */
    public void close() {}
    

    // TODO:
    // In the database, need to ensure we have else
    // create collection (table in RDBMS) websites, create collection webpages.
    // The webpages collection will have sentences embedded based on my decisions from
    // reading the series
    // https://www.mongodb.com/blog/post/6-rules-of-thumb-for-mongodb-schema-design-part-1    
    // Then need functions:
    // insertWebsiteDocument()
    // insertWebpageDocument()
    
    public static void main(String args[]) {
    try {
        MongoDBAccess mongodbCon = new MongoDBAccess();
        mongodbCon.connectToDB();
        mongodbCon.showCollections();

    } catch(Exception e) {
        e.printStackTrace();
    }
    }
}