Changeset 33871

Timestamp:

2020-01-24T20:59:42+13:00 (4 years ago)

Author:

ak19

Message:

Removed mostly duplicated older version of method but left the different parts commented out inside the new method. Improved regex in mongodb find query.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (4 diffs)
• RandomURLsForDomainGenerator.java (modified) (2 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -276 +276 @@
      * http://pingax.com/trick-convert-mongo-shell-query-equivalent-java-objects/
 */
-    public ArrayList<String> oldWorks_queryAllMatchingIsMRIURLs(String domain) {
+    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
     
     final ArrayList<String> urlsList = new ArrayList<String>();
@@ -285 +285 @@
 
     // load the "webpages" db table
-    // in mongodb, the equivalent of db tables are called 'collections'
-    
-    
-    //Pattern pattern = Pattern.compile(".*"+domain+".*");
-
-    // escape dots in domain for regex
-    String pattern = "/"+domain.replace(".", "\\.")+"/";
-    
+    // in mongodb, the equivalent of db tables are called 'collections' 
     MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
 
-
+    // code we'll execute in Iterable.forEach() below
     Block<Document> storeURL = new Block<Document>() {
         @Override
@@ -302 +295 @@
             String url = document.getString("URL");
             // add to our urlsList
-            System.out.println(url);
+            //System.out.println(url);
             urlsList.add(url);
         }
@@ -309 +302 @@
 
     
-    // do mongodb query:
-    // test example:
-    //collection.find(eq("isMRI", true)).first();
-    //
-    // db.getCollection('Webpages').find({URL: /domain/, isMRI: true}, {URL: 1, _id: 0})
-    collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
-
-    
-    return urlsList;
-    }
-
-    public ArrayList<String> queryAllMatchingIsMRIURLs(String domain) {
-    
-    final ArrayList<String> urlsList = new ArrayList<String>();
-    
-    // remove any http(s)://(www.) from the start of URL first
-    // since it goes into a regex
-    domain = Utility.stripProtocolAndWWWFromURL(domain);
-
-    // load the "webpages" db table
-    // in mongodb, the equivalent of db tables are called 'collections'
-    
-    
-    //Pattern pattern = Pattern.compile(".*"+domain+".*");
-
-    // escape dots in domain for regex
-    String pattern = "/"+domain.replace(".", "\\.")+"/";
-    
-    MongoCollection<Document> collection = this.database.getCollection(WEBPAGES_COLLECTION);
-
-
-    Block<Document> storeURL = new Block<Document>() {
-        @Override
-        public void apply(final Document document) {
-            //System.out.println(document.toJson());
-            String url = document.getString("URL");
-            // add to our urlsList
-            System.out.println(url);
-            urlsList.add(url);
-        }
-        };
-
-
-    
-    // do mongodb query
-    // db.getCollection('Webpages').find({URL: /domain/, isMRI: true}, {URL: 1, _id: 0})
+    // Run the following mongodb query:
+    //    db.getCollection('Webpages').find({URL: /domain/, isMRI: true}, {URL: 1, _id: 0})
+    
+    // 1. One way that works:
+    //collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
+
+    // 2. Another way:
     String query = "{URL: /DOMAIN/, isMRI: true}";
+    domain = domain.replace(".", "\\."); // escape dots in domain for regex
     query = query.replace("DOMAIN", domain);
+
+    //System.err.println("Executing find query: " + query);
     
     BasicDBObject findObj = BasicDBObject.parse(query);
     BasicDBObject projectionObj = BasicDBObject.parse("{URL: 1, _id: 0}");
     
-    //collection.find(and(eq("isMRI", true), regex("URL", pattern))).projection(fields(include("URL"), excludeId())).forEach(storeURL);
+
     collection.find(findObj).projection(projectionObj).forEach(storeURL);
     

other-projects/maori-lang-detection/src/org/greenstone/atea/RandomURLsForDomainGenerator.java

@@ -18 +18 @@
  * TO RUN:
  *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*" org/greenstone/atea/RandomURLsForDomainGenerator ../domains.txt
+ *       java -cp ".:../conf:../lib/*" org/greenstone/atea/RandomURLsForDomainGenerator ../mongodb-data/domainsNZ_IsMRI.txt  255
  *
 */
@@ -65 +65 @@
     String fileName = domainsFile.getName();
     File outFile = new File(parentFolder, "random"+numURLs+"_"+fileName);
-    File fullSetOutFile = new File(parentFolder, "all_"+fileName);
+    //File fullSetOutFile = new File(parentFolder, "allPages_"+fileName);
     
-    // shuffle list and take the first n
+    // shuffle list and take the first n - write to file
     try (
          Writer writer = new BufferedWriter(new FileWriter(outFile));