Index: /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt
===================================================================
--- /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/MoreReading/crawling-Nutch.txt	(revision 33623)
@@ -492,4 +492,13 @@
 
 
+INSTALLATION MONGO-DB AND CLIENT
+FROM: https://docs.mongodb.com/manual/tutorial/install-mongodb-on-ubuntu/
+	wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | sudo apt-key add -
+	echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.2.list
+	sudo apt-get update
+	sudo apt-get install -y mongodb-org
+
+UNINSTALLING
+	https://www.anintegratedworld.com/uninstall-mongodb-in-ubuntu-via-command-line-in-3-easy-steps/
 
 
Index: /gs3-extensions/maori-lang-detection/conf/config.properties
===================================================================
--- /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/conf/config.properties	(revision 33623)
@@ -23,6 +23,9 @@
 
 
-mongodb.user=admin
-mongodb.pwd=pinky
-#CHANGEME
-
+mongodb.user=anupama
+mongodb.pwd=chang3m3
+# default mongodb port is 27017. Don't change the port unless you really have configured
+# your mongodb server to listen at some other port
+mongodb.port=27017
+mongodb.host=mongodb.cms.waikato.ac.nz
+mongodb.dbname=ateacrawldata
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java	(revision 33623)
@@ -237,26 +237,4 @@
     }
 
-    /** Work out the 'domain' for a given url.
-     * This retains any www. or subdomain prefix.
-     */
-    public static String getDomainForURL(String url, boolean withProtocol) {
-	int startIndex = startIndex = url.indexOf("//"); // for http:// or https:// prefix
-	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
-	// the keep the URL around in case param withProtocol=true
-	String protocol = (startIndex == -1) ? "" : url.substring(0, startIndex);
-	
-	String domain = url.substring(startIndex);
-	int endIndex = domain.indexOf("/");
-	if(endIndex == -1) endIndex = domain.length();
-	domain = domain.substring(0, endIndex);
-
-	if(withProtocol) {
-	    // now that we have the domain (everything to the first / when there is no protocol)
-	    // can glue the protocol back on
-	    domain = protocol + domain;
-	}
-	
-	return domain;
-    }
 
     /** Utility function to help escape regex characters in URL to go into regex-urlfilter.txt */
@@ -304,5 +282,5 @@
 		// work out domain. This retains any www. or subdomain prefix
 		// passing true to further also retain the http(s) protocol
-		domainWithProtocol = getDomainForURL(url, true);
+		domainWithProtocol = Utility.getDomainForURL(url, true);
 
 		Set<String> urlsSet;
@@ -316,4 +294,5 @@
 		}
 
+		/*
 		// Dr Nichols said that a url that was located outside the country and
 		// which had /mi/ URLs was more likely to be an autotranslated (product) site.
@@ -322,6 +301,7 @@
 		// then add that domain (if not already added) and that url into a file
 		// for later manual inspection
-		if(!domainWithProtocol.endsWith(".nz") && (url.contains("/mi/") || url.endsWith("/mi"))) {
-		    /*
+		if(!domainWithProtocol.endsWith(".nz")
+		   && (url.contains("/mi/") || url.endsWith("/mi"))) {
+		    
 		    if(!possibleProductDomains.contains(domainWithProtocol)) {
 
@@ -345,13 +325,15 @@
 			if(!isInNZ) {
 			    possibleProductDomains.add(domainWithProtocol);
-			    // write both domain and a sample URL on that site out to file
+			    // write both domain and a sample seedURL on that site out to file
 			    possibleProductSitesWriter.write(countryCode + " : " + domainWithProtocol + "\n");			    
 			    possibleProductSitesWriter.write("\t" + url + "\n");
 			}
-			}*/ /*else {
-			// already wrote out domain to file at some point, write just the URL out to file
-			possibleProductSitesWriter.write("\t" + url + "\n");
-			}*/
-		}
+		    }
+		    //else {
+		    // already wrote out domain to file at some point, write just the URL out to file
+		    //possibleProductSitesWriter.write("\t" + url + "\n");
+		    //}
+		}
+		*/
 	    }
 	} catch (IOException ioe) {
@@ -686,5 +668,5 @@
 	// if any portion of the URL contains the word "livejasmin", or even "jasmin" actually,
 	// then it's an adult site, so blacklist the entire domain if it wasn't already blacklisted
-	String domainWithoutProtocol = getDomainForURL(url, false); // remove protocol
+	String domainWithoutProtocol = Utility.getDomainForURL(url, false); // remove protocol
 	if(!isBlackListed && url.contains("jasmin")) {
 	    logger.warn("### Blacklisting additional domain (likely an adult site): " + domainWithoutProtocol);
@@ -821,5 +803,5 @@
     public static void printUsage() {
 	System.err.println("Run this program as:");
-	System.err.println("\tCCWetProcessor <path to 'ccrawl-data' folder> <output folder path>");	
+	System.err.println("\tCCWetProcessor <path to 'ccrawl-data' input folder> <output folder path>");	
     }
 
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java	(revision 33623)
@@ -2,7 +2,10 @@
 
 
+import com.mongodb.client.MongoCollection; 
 import com.mongodb.client.MongoDatabase; 
 import com.mongodb.MongoClient; 
 import com.mongodb.MongoCredential;  
+
+import org.bson.Document;
 
 import java.io.BufferedReader;
@@ -23,4 +26,20 @@
  * TO RUN:
  *       java -cp ".:../conf:../lib/*" org.greenstone.atea.MongoDBAccess
+ *
+ * Manually connecting to mongodb from client:
+ *    mongo 'mongodb://mongodb.cms.waikato.ac.nz:27017' -u USERNAME -p
+ * Then after connecting with pwd, type:
+ *    use DBNAME
+ *
+ * Or connect to mongodb and specify db in one statement:
+ *    mongo 'mongodb://mongodb.cms.waikato.ac.nz:27017/DBNAME?authSource=admin' -u USERNAME -p
+ *
+ * Some links:
+ *   - https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection
+ *   - https://docs.mongodb.com/manual/reference/glossary/ (particularly "collection")
+ *   - https://tecadmin.net/tutorial/mongodb/drop-collection/
+ * IMPORTANT LINK:
+ *   - https://www.mongodb.com/blog/post/6-rules-of-thumb-for-mongodb-schema-design-part-1
+ *
  */
 public class MongoDBAccess {
@@ -28,8 +47,8 @@
     private static Logger logger = Logger.getLogger(org.greenstone.atea.MongoDBAccess.class.getName());
     
-    final static String HOST = "localhost";
-    final static int PORT = 27017; // mongodb port
-    final static String PROPS_FILENAME = "config.properties";
-    final static String DB_NAME = "ateacrawldata";
+    String HOST = "localhost";
+    int PORT = 27017; // mongodb port
+    String PROPS_FILENAME = "config.properties";
+    String DB_NAME = "ateacrawldata";
     
     private String USERNAME;
@@ -37,4 +56,7 @@
 
 
+    private MongoClient mongo = null;
+    private MongoDatabase database = null;
+    
     public MongoDBAccess() throws Exception {
 	boolean success = false;
@@ -55,5 +77,5 @@
 	if(USERNAME.equals("")) {
 	    USERNAME = "root";
-	    logger.warn("WARNING: No sensible value for mongodb.user specified in " + PROPS_FILENAME + " defaulting to: " + USERNAME);
+	    logger.warn("WARNING: No sensible value for mongodb.user specified in " + PROPS_FILENAME + ". Attempting to use: " + USERNAME);
 			}
 	PASSWORD = props.getProperty("mongodb.pwd");
@@ -66,10 +88,24 @@
 	    throw new Exception("************ FATAL ERROR: Change DB password in properties file " + PROPS_FILENAME);	    
 	}
+
+	HOST = props.getProperty("mongodb.host", HOST);
+	String port = props.getProperty("mongodb.port", Integer.toString(PORT));
+	PORT = Integer.parseInt(port);
+	DB_NAME = props.getProperty("mongodb.dbname", DB_NAME);
+
+	logger.info("Connecting to mongodb with:");
+	logger.info(" - host:    " + HOST);
+	logger.info(" - port:    " + PORT);
+	logger.info(" - user:    " + USERNAME);
+	logger.info(" - db name: " + DB_NAME);	
     }
 
-
+    /** 
+     * Since we have only a single MongoClient, don't need to call close/disconnect on it as per
+     * https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection
+     */
     public void connectToDB() throws Exception {
 	// Creating a Mongo client 
-	MongoClient mongo = new MongoClient( HOST, PORT ); 
+	mongo = new MongoClient( HOST, PORT ); 
 	
 	// Creating Credentials 
@@ -79,13 +115,23 @@
 	
 	// Accessing the database 
-	MongoDatabase database = mongo.getDatabase(DB_NAME); 
-	//System.out.println("Credentials: "+ credential);
+	database = mongo.getDatabase(DB_NAME); 
+	logger.info("Credentials: "+ credential);
     }
     
 
+    /*
+    public void insertDocument() {
+	MongoCollection<Document> collection = this.database.getCollection("sampleCollection");
+    }
+    */
+
+    // create collection (table in RDBMS) websites, create collection webpages
+    // webpages collection will have sentences embedded
+    
     public static void main(String args[]) {
 	try {
 	    MongoDBAccess mongodbCon = new MongoDBAccess();
-	    //mongodbCon.connectToDB();
+	    mongodbCon.connectToDB();
+	    //mongodbCon.insertDocument();
 	}catch(Exception e) {
 	    e.printStackTrace();
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33623)
@@ -182,5 +182,5 @@
 	    TextDumpPage firstPage = pages.get(0);
 	    String url = firstPage.getPageURL();
-	    this.domainOfSite = CCWETProcessor.getDomainForURL(url, true);
+	    this.domainOfSite = Utility.getDomainForURL(url, true);
 	}
 	else {
@@ -248,5 +248,6 @@
 		
 		page.addMRILanguageStatus(isMRI);
-
+		
+    
 		// Even if the entire page is not found to be overall in MÄori,
 		// let's still inspect the sentences of the page and count how many (if any)
@@ -281,5 +282,10 @@
 		    webpageCSVPrinter.printRecord(WEBPAGE_COUNTER++,
 						  SITE_COUNTER, /* alternative: this.siteID */
-						  url, isMRI, totalSentences, numSentencesInMRI);
+						  url,
+						  //"origCharEncoding", "modifiedTime", "fetchTime",
+						  page.getOriginalCharEncoding(),
+						  page.getModifiedTime(),
+						  page.getFetchTime(),
+						  isMRI, totalSentences, numSentencesInMRI);
 
 		    // Write the sentences that are in te reo into the mri-sentences CSV file
@@ -393,6 +399,7 @@
 		   "domainURL","totalPagesInSite", "numPagesInMRI", "numOtherPagesContainingMRI",
 		   "nutchCrawlTimestamp", "crawlUnfinished", "redoCrawl");
-	    webpagesCSVPrinter.printRecord("webpageID", "websiteID", "URL", "isMRI",
-					   "numSentences", "numSentencesInMRI");
+	    webpagesCSVPrinter.printRecord("webpageID", "websiteID", "URL",
+					   "origCharEncoding", "modifiedTime", "fetchTime",
+					   "isMRI", "numSentences", "numSentencesInMRI");
 	    mriSentencesCSVPrinter.printRecord("sentenceID", "webpageID", "sentence");
 	    
@@ -435,5 +442,5 @@
 	    
 	} catch(Exception e) {
-	    // can get an exception when instantiating CCWETProcessor instance
+	    // can get an exception when instantiating NutchTextDumpProcessor instance
 	    // or with CSV file
 	    logger.error(e.getMessage(), e);
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33623)
@@ -84,4 +84,8 @@
 			    String k = line.substring(0, endIndex);
 			    String v = line.substring(endIndex+1);
+			    if(k.startsWith("metadata")) {
+				k = k.substring("metadata".length());
+			    }
+				
 			    tuples.put(k.trim(), v.trim());
 			} else {
@@ -134,4 +138,26 @@
     }
 
+    /* Dr Nichols suggested storing timestamp and char encoding. Not sure which timestamp
+       or encoding he meant, but storing 2 of several timestamps and selecting
+       original character encoding (presumably the char encoding of the page) out of 2
+       pieces of char encoding metadata to store. */
+    public String getModifiedTime() {	
+	// is this the webpage's last mod time?	
+	String time = tuples.get("modifiedTime");
+	time = time.equals("0") ? "" : time; // zero will be assumed to be epoch, rather than unset
+	return time;
+    }    
+    public String getFetchTime() {
+	// is this the nutch crawl time
+	String time = tuples.get("fetchTime");
+	time = time.equals("0") ? "" : time; // zero will be assumed to be epoch, rather than unset
+	return time;
+	
+    } 
+    public String getOriginalCharEncoding() {
+	// is this the web page's char-encoding?
+	return tuples.get("OriginalCharEncoding");
+    }
+    
     public String get(String key) {
 	return tuples.get(key);
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33622)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/Utility.java	(revision 33623)
@@ -82,4 +82,27 @@
     }
     
+    /** Work out the 'domain' for a given url.
+     * This retains any www. or subdomain prefix.
+     */
+    public static String getDomainForURL(String url, boolean withProtocol) {
+	int startIndex = startIndex = url.indexOf("//"); // for http:// or https:// prefix
+	startIndex = (startIndex == -1) ? 0 : (startIndex+2); // skip past the protocol's // portion
+	// the keep the URL around in case param withProtocol=true
+	String protocol = (startIndex == -1) ? "" : url.substring(0, startIndex);
+	
+	String domain = url.substring(startIndex);
+	int endIndex = domain.indexOf("/");
+	if(endIndex == -1) endIndex = domain.length();
+	domain = domain.substring(0, endIndex);
+
+	if(withProtocol) {
+	    // now that we have the domain (everything to the first / when there is no protocol)
+	    // can glue the protocol back on
+	    domain = protocol + domain;
+	}
+	
+	return domain;
+    }
+    
     public static boolean isDomainInCountry(String domainWithProtocol,
 					    String countryCode, File geoLiteCityDatFile)
