Changeset 33870

Timestamp:

2020-01-24T20:48:17+13:00 (4 years ago)

Author:

ak19

Message:

Got the mongodb query working in Java in 2 different ways: the fully Java way and the way where I parse the query and parse the projection portion and then run the mongodb collection.find().projection() on it. Both work now. And thed RandomURLsForDomainGenerator now works fine.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (5 diffs)
• RandomURLsForDomainGenerator.java (modified) (1 diff)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -270 +270 @@
      * Java mongodb projection: https://stackoverflow.com/questions/44894497/retrieving-data-with-mongodb-java-driver-3-4-using-find-method-with-projection
      * mongodb projection: https://docs.mongodb.com/v3.2/reference/method/db.collection.find/#db.collection.find
+     *
+     * Parse MongoDB query into Java: https://stackoverflow.com/questions/17326747/parsing-strings-to-mongodb-query-documents-with-operators-in-java
+     * Maybe also https://stackoverflow.com/questions/48000891/parse-mongodb-json-query-in-java-with-multiple-criteria
+     * https://stackoverflow.com/questions/55029222/parse-mongodb-query-to-java
+     * http://pingax.com/trick-convert-mongo-shell-query-equivalent-java-objects/
 */
-    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
+    public ArrayList<String> oldWorks_queryAllMatchingIsMRIURLs(String domain) {
     
     final ArrayList<String> urlsList = new ArrayList<String>();
+    
+    // remove any http(s)://(www.) from the start of URL first
+    // since it goes into a regex
+    domain = Utility.stripProtocolAndWWWFromURL(domain);
 
     // load the "webpages" db table
@@ -280 +289 @@
     
     //Pattern pattern = Pattern.compile(".*"+domain+".*");
+
+    // escape dots in domain for regex
     String pattern = "/"+domain.replace(".", "\\.")+"/";
     
@@ -291 +302 @@
             String url = document.getString("URL");
             // add to our urlsList
+            System.out.println(url);
             urlsList.add(url);
         }
@@ -301 +313 @@
     //collection.find(eq("isMRI", true)).first();
     //
-    // db.getCollection('Webpages').find({URL:/.*domain.*/, isMRI: true}, {URL: 1, _id: 0})
+    // db.getCollection('Webpages').find({URL: /domain/, isMRI: true}, {URL: 1, _id: 0})
     collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
 
@@ -307 +319 @@
     return urlsList;
     }
+
+    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
+    
+    final ArrayList<String> urlsList = new ArrayList<String>();
+    
+    // remove any http(s)://(www.) from the start of URL first
+    // since it goes into a regex
+    domain = Utility.stripProtocolAndWWWFromURL(domain);
+
+    // load the "webpages" db table
+    // in mongodb, the equivalent of db tables are called 'collections'
+    
+    
+    //Pattern pattern = Pattern.compile(".*"+domain+".*");
+
+    // escape dots in domain for regex
+    String pattern = "/"+domain.replace(".", "\\.")+"/";
+    
+    MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
+
+
+    Block<Document> storeURL = new Block<Document>() {
+        @Override
+        public void apply(final Document document) {
+            //System.out.println(document.toJson());
+            String url = document.getString("URL");
+            // add to our urlsList
+            System.out.println(url);
+            urlsList.add(url);
+        }
+        };
+
+
+    
+    // do mongodb query
+    // db.getCollection('Webpages').find({URL: /domain/, isMRI: true}, {URL: 1, _id: 0})
+    String query = "{URL: /DOMAIN/, isMRI: true}";
+    query = query.replace("DOMAIN", domain);
+    
+    BasicDBObject findObj = BasicDBObject.parse(query);
+    BasicDBObject projectionObj = BasicDBObject.parse("{URL: 1, _id: 0}");
+    
+    //collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
+    collection.find(findObj).projection(projectionObj).forEach(storeURL);
+    
+    return urlsList;
+    }
+
+    
     
     /** https://stackoverflow.com/questions/19938153/do-i-need-to-explicitly-close-connection */

other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java

@@ -60 +60 @@
     }
 
-    // copy into array
-    /*
-    urls = new String[urlsList.size()];
-    String[] urls = urlsList.toArray(urls);
-    urlsList.clear();
-    */
-
-    /*
-    // 2. generate numURLs of UNIQUE numbers between 0 to urls.length
-    // https://stackoverflow.com/questions/8115722/generating-unique-random-numbers-in-java
-    // https://www.geeksforgeeks.org/iterator-vs-foreach-in-java/
-    // BETTER: https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
-    
-    
-    // 3. then for each number, write the url at that index in array urls into file.
-    */
-
-    // Shuffle the urlsList, then write out the first numURLs into a file.
-    
-    // BETTER: https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
+    // Shuffle the urlsList, then write out the first numURLs into a file.  
+    // https://stackoverflow.com/questions/5505927/how-to-generate-a-random-permutation-in-java
     File parentFolder = domainsFile.getParentFile();
     String fileName = domainsFile.getName();
     File outFile = new File(parentFolder, "random"+numURLs+"_"+fileName);
+    File fullSetOutFile = new File(parentFolder, "all_"+fileName);
     
     // shuffle list and take the first n