Changeset 33882 for other-projects

Timestamp:

2020-01-30T22:54:39+13:00 (4 years ago)

Author:

ak19

Message:

Code now writes both a listing of all non-autotranslated websites and a listing of overseas autotranslated sites.

Location:

other-projects/maori-lang-detection/src/org/greenstone/atea

Files:

• MongoDBAccess.java (modified) (4 diffs)
• WebPageURLsListing.java (modified) (5 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/MongoDBAccess.java

@@ -96 +96 @@
     public static final int CONTAINS_MRI = 1;
 
+    /** Some reused fieldnames in the Websites collection */
+    private static final String FILTER_NUMPAGES_IN_MRI = "numPagesInMRI";
+    private static final String FILTER_NUMPAGES_CONTAINING_MRI = "numPagesContainingMRI";
+
     // configuration details, some with fallback values
     private String HOST = "localhost";
@@ -402 +406 @@
     ]);
     */
-    public void aggregateContainsMRIForNZ(Writer writer) throws IOException {
+    public void aggregateContainsMRIForNZ(Writer writer, int filterType) throws IOException {
     // working with the WebSites collection, not WebPages collection!
     MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
 
-    
-    //String isMRI_filter = 
+    String mriFilterString = (filterType == CONTAINS_MRI) ? "{numPagesContainingMRI: {$gt: 0}}" : "{numPagesInMRI: {$gt: 0}}";
     
     Bson orQuery = or(
@@ -414 +417 @@
               );
     Bson andQuery = and(
-        BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+        BasicDBObject.parse(mriFilterString),
         orQuery);
     
@@ -456 +459 @@
     ]);
     */
-    public void aggregateContainsMRIForOverseas(Writer writer) throws UncheckedIOException {
+    public void aggregateContainsMRIForOverseas(Writer writer, int filterType,
+                        boolean isMiInURLPath) throws UncheckedIOException
+    {
     // working with the WebSites collection, not WebPages collection!
     MongoCollection<Document> collection = this.database.getCollection(WEBSITES_COLLECTION);
-
+    
+    String mriFilterString = (filterType == CONTAINS_MRI) ? "{numPagesContainingMRI: {$gt: 0}}" : "{numPagesInMRI: {$gt: 0}}";
     
     Bson orQuery = or(
               BasicDBObject.parse("{geoLocationCountryCode: \"AU\"}"),
-              BasicDBObject.parse("{urlContainsLangCodeInPath: false}")
+              BasicDBObject.parse("{urlContainsLangCodeInPath: "+ isMiInURLPath +"}")
+              // e.g. "{urlContainsLangCodeInPath: false}"
               );
     Bson andQuery = and(
         BasicDBObject.parse("{geoLocationCountryCode: {$ne: \"NZ\"}}"),
         BasicDBObject.parse("{domain: {$not: /\\.nz/}}"),
-        BasicDBObject.parse("{numPagesContainingMRI: {$gt: 0}}"),
+        BasicDBObject.parse(mriFilterString),
         orQuery);
 

other-projects/maori-lang-detection/src/org/greenstone/atea/WebPageURLsListing.java

@@ -25 +25 @@
     
     private final MongoDBAccess mongodbAccess;
-    private int numURLs;
-    private File domainsFile;
-
-
-    
-    public WebPageURLsListing(MongoDBAccess mongodbAccess,
-                    File domainsFile)
+    private File outFolder;
+
+    public WebPageURLsListing(MongoDBAccess mongodbAccess, File outFolder)
     {
     this.mongodbAccess = mongodbAccess;
-    this.domainsFile = domainsFile;
-    }
-    
-    public WebPageURLsListing(MongoDBAccess mongodbAccess,
-                    File domainsFile,
-                    int numURLs)
-    {
-    this(mongodbAccess, domainsFile);
-    this.numURLs = numURLs;
-    }
-
-    public String produceURLsForPagesInMRI() {
-    return writeFile(MongoDBAccess.IS_MRI);
-    }
-    
-    public String produceURLsForPagesContainingMRI() {
-    return writeFile(MongoDBAccess.CONTAINS_MRI);
-    }
-
-
-    public String writeFile(int filterType) {
+    this.outFolder = outFolder;
+    }
+    
+    public String produceURLsForPagesInMRI(File domainsFile) {
+    return writeFile(MongoDBAccess.IS_MRI, domainsFile);
+    }
+    
+    public String produceURLsForPagesContainingMRI(File domainsFile) {
+    return writeFile(MongoDBAccess.CONTAINS_MRI, domainsFile);
+    }
+
+
+    public String writeFile(int filterType, File domainsFile/*, int numURLs*/) {
 
     ArrayList<String> urlsList = new ArrayList<String>();
@@ -126 +114 @@
 
     /** 
-     * Create the file 
+     * Create the file 5counts_tentativeNonAutotranslatedSites.json
+     * that contains the count and domains for NZ sites (NZ origin or nz TLD) that CONTAIN_MRI
+     * followed by counts and domain listing for overseas sites that are either from Australia
+     * or don't contain mi in their URL path.
      * @return full path of file generated
      */
     public String writeTentativeNonAutotranslatedSites() {
-    File outFolder = new File("../mongodb-data/").getAbsoluteFile();
-    File outFile = new File(outFolder, "5counts_tentativeNonAutotranslatedSites.json");
+    
+    File outFile = new File(outFolder, "5a_counts_tentativeNonAutotranslatedSites.json");
 
     String filename = outFile.getAbsolutePath();
@@ -139 +130 @@
          ) {
         // first write out NZ sites and .nz TLD count and domains
-        mongodbAccess.aggregateContainsMRIForNZ(writer);
-        // next write out all overseas sites and .nz TLD count and domains
-        mongodbAccess.aggregateContainsMRIForOverseas(writer);
+        mongodbAccess.aggregateContainsMRIForNZ(writer, MongoDBAccess.CONTAINS_MRI);
+        // next write out all overseas sites (not NZ origin or .nz TLD)
+        // that have no "mi" in the URL path as mi.* or */mi
+        boolean isMiInURLPath = false;
+        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
 
         filename = outFile.getCanonicalPath();
@@ -148 +141 @@
         logger.error(e.getMessage(), e);
     }
+    
+    System.err.println("*** Wrote file: " + filename);
 
     return filename;
     }
-    
+
+    /**
+     * Listing of the remainder of overseas sites that CONTAIN_MRI not included by
+     * writeTentativeNonAutotranslatedSites(): those that have mi in their URL path
+     */
+    public String writeOverseasSitesWithMiInURLPath() {
+    File outFile = new File(outFolder, "5b_counts_overseasSitesWithMiInPath.json");
+
+    String filename = outFile.getAbsolutePath();
+    try (
+         Writer writer = new BufferedWriter(new FileWriter(outFile));
+         ) {
+        boolean isMiInURLPath = true;
+        mongodbAccess.aggregateContainsMRIForOverseas(writer, MongoDBAccess.CONTAINS_MRI, isMiInURLPath);
+        filename = outFile.getCanonicalPath();
+    } catch(Exception e) {
+        logger.error("Unable to write to file " + outFile.getAbsolutePath());
+        logger.error(e.getMessage(), e);
+    }
+
+    System.err.println("*** Wrote file: " + filename);
+    return filename;
+    }
     
     public static void printUsage() {
@@ -170 +187 @@
          MongoDBAccess mongodb = new MongoDBAccess();
          ) {
-        File domainsFile = new File(args[0]);
-        if(!domainsFile.exists()) {
-        System.err.println("File " + domainsFile + " does not exist");
-        System.exit(-1);
-        }
-
-        //int genNumURLs = Integer.parseInt(args[1]);
 
         mongodb.connectToDB();
-        
-        WebPageURLsListing listing = new WebPageURLsListing(mongodb, domainsFile);
-        //String isMRIFile = listing.produceURLsForPagesInMRI();
-        //String containsMRIFile = listing.produceURLsForPagesContainingMRI();
-        String filename = listing.writeTentativeNonAutotranslatedSites();
-        System.err.println("Check file: " + filename);
+
+        // output files will be stored in mongodb-data-auto
+        File outFolder = new File("../mongodb-data-auto/").getAbsoluteFile();
+        WebPageURLsListing listing = new WebPageURLsListing(mongodb, outFolder);
+
+        
+        if(args.length >= 1) {
+        File domainsFile = new File(args[0]);
+        if(!domainsFile.exists()) {
+            System.err.println("File " + domainsFile + " does not exist");
+            System.exit(-1);
+        }
+
+        
+        //int genNumURLs = Integer.parseInt(args[1]);
+        //String isMRIFile = listing.produceURLsForPagesInMRI(domainsFile);
+        //String containsMRIFile = listing.produceURLsForPagesContainingMRI(domainsFile);
+
+        //listing.writeWebPagesOfAllNZSitesAndDomainListing();
+        
+        } else {        
+
+        String filename = listing.writeTentativeNonAutotranslatedSites();
+        filename = listing.writeOverseasSitesWithMiInURLPath();
+        }