Changeset 33869

Timestamp:

2020-01-23T22:59:46+13:00 (4 years ago)

Author:

ak19

Message:

First cut at the RandomURLsForDomainGenerator.java class and the mongodb method it needs added to MongoDBAccess. Still need to generate a domainURLs file to start testing whether the code even works. But at least it finally compiles.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• CountryCodeCountsMapData.java (modified) (1 diff)
• MongoDBAccess.java (modified) (5 diffs)
• RandomURLsForDomainGenerator.java (added)

other-projects/maori-lang-detection/src/org/greenstone/atea/CountryCodeCountsMapData.java

@@ -601 +601 @@
     try {
         File countsFile = new File(args[0]);
-        
+        if(!countsFile.exists()) {
+        System.err.println("File " + countsFile + " does not exist");
+        System.exit(-1);
+        }
+        
         CountryCodeCountsMapData mapData = new CountryCodeCountsMapData(args[0]);
 

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -2 +2 @@
 
 //import org.bson.BSONObject;
-    
+
 import com.mongodb.client.MongoCollection; 
 import com.mongodb.client.MongoDatabase; 
 //import com.mongodb.client.MongoIterable;
+
+// to use collection.find() filters like eq(), regex() etc
+import static com.mongodb.client.model.Filters.*;
+// to use collection.find().projection() filters like include() etc
+import static com.mongodb.client.model.Projections.*;
+
+//import org.bson.conversions.Bson;
 import com.mongodb.BasicDBObject;
 import com.mongodb.MongoClient; 
@@ -11 +18 @@
 import com.mongodb.ServerAddress;
 import com.mongodb.MongoClientOptions;
+
+import com.mongodb.Block;
 
 import org.bson.Document;
@@ -20 +29 @@
 import java.util.List;
 import java.util.Properties;
-
+import java.util.regex.Pattern;
 
 import org.apache.log4j.Logger;
@@ -58 +67 @@
     
     static final String PROPS_FILENAME = "config.properties";
-    public static final String WEBPAGES_COLLECTION = "webpages";
-    public static final String WEBSITES_COLLECTION = "websites";    
+    public static final String WEBPAGES_COLLECTION = "Webpages";
+    public static final String WEBSITES_COLLECTION = "Websites";    
 
     // configuration details, some with fallback values
@@ -256 +265 @@
     */
 
+    /** 
+     * Java mongodb find: https://mongodb.github.io/mongo-java-driver/3.4/driver/getting-started/quick-start/ 
+     * Java mongodb find filters: https://mongodb.github.io/mongo-java-driver/3.4/javadoc/?com/mongodb/client/model/Filters.html
+     * Java mongodb projection: https://stackoverflow.com/questions/44894497/retrieving-data-with-mongodb-java-driver-3-4-using-find-method-with-projection
+     * mongodb projection: https://docs.mongodb.com/v3.2/reference/method/db.collection.find/#db.collection.find
+*/
+    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
+    
+    final ArrayList<String> urlsList = new ArrayList<String>();
+
+    // load the "webpages" db table
+    // in mongodb, the equivalent of db tables are called 'collections'
+    
+    
+    //Pattern pattern = Pattern.compile(".*"+domain+".*");
+    String pattern = "/"+domain.replace(".", "\\.")+"/";
+    
+    MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
+
+
+    Block<Document> storeURL = new Block<Document>() {
+        @Override
+        public void apply(final Document document) {
+            //System.out.println(document.toJson());
+            String url = document.getString("URL");
+            // add to our urlsList
+            urlsList.add(url);
+        }
+        };
+
+
+    
+    // do mongodb query:
+    // test example:
+    //collection.find(eq("isMRI", true)).first();
+    //
+    // db.getCollection('Webpages').find({URL:/.*domain.*/, isMRI: true}, {URL: 1, _id: 0})
+    collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
+
+    
+    return urlsList;
+    }
+    
     /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */
     public void close() {}