Changeset 33519 for gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

Timestamp:

2019-09-24T21:40:16+12:00 (5 years ago)

Author:

ak19

Message:

Code still writes out the global seedURLs.txt and regex-urlfilter.txt (in case this remains meaningful), but now also creates individual site directories containing their individual seedURLs.txt and regex-urlfilter.txt

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java (modified) (1 diff)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/CCWETProcessor.java

@@ -244 +244 @@
     */
 
+    int domainCount = 0;
+    File sitesFolder = new File(outputFolder, "sites");
+    if(!sitesFolder.exists()) {
+        sitesFolder.mkdir();
+    }
+    final String FORMATSTR = "%05d";
+    
     // write out each domain followed in sequence by all urls we found in that domain
     // (urls with tab up front)
-    try (BufferedWriter seedURLsWriter = new BufferedWriter(new FileWriter(seedURLsFile))) {
+    try (
+         BufferedWriter seedURLsWriter = new BufferedWriter(new FileWriter(seedURLsFile));
+         BufferedWriter urlFilterWriter = new BufferedWriter(new FileWriter(urlFilterFile))
+         ) {
         //Set<Map.Entry<String, Set<String>>> domainsSet = domainsToURLsMap.keySet();
         Set<String> domainsSet = domainsToURLsMap.keySet();
         Iterator<String> domainIterator = domainsSet.iterator();
-
+        
         while(domainIterator.hasNext()) {
+        domainCount++;
+        String siteID = String.format(FORMATSTR, domainCount);
+        File domainFolder = new File(sitesFolder, siteID);
+        domainFolder.mkdir();
+        
         // write out the domain
         String domain = domainIterator.next();
-        seedURLsWriter.write(domain + "\n");
-        
-        // next write out the urls for the domain with a tab prefixed to each
-        Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
-        Iterator<String> urlIterator = urlsForDomainSet.iterator();
-        while(urlIterator.hasNext()) {
-            String url = urlIterator.next();
-            seedURLsWriter.write("\t" + url + "\n");
-        }       
+        //seedURLsWriter.write(domain + "\n");
+        // nutch.apache.org => +^https?://([a-z0-9-]+\.)*nutch\.apache\.org/
+        String regexed_domain = FILTER_REGEX_PREFIX + domain.replace(".", "\\.") + "/";
+        urlFilterWriter.write(regexed_domain + "\n");
+
+        // for every domain, we need sites/0000x/ folder containing its own
+        // INDIVIDUAL seedURLs.txt and regex-urlfilter.txt
+        // We still have a global seedURLs.txt and regex-urlfilter.txt too.
+        File siteSeedsFile = new File(domainFolder, "seedURLs.txt"); // e.g. sites/00001/seedURLs.txt
+        File siteRegexFile = new File(domainFolder, "regex-urlfilter.txt"); // e.g. sites/00001/regex-urlfilter.txt
+        try (
+             BufferedWriter siteURLsWriter = new BufferedWriter(new FileWriter(siteSeedsFile));
+             BufferedWriter siteRegexWriter = new BufferedWriter(new FileWriter(siteRegexFile));
+             ) {
+            // only write urls and no domain into single global seedurls file
+            // But write domain and tabbed urls into individual sites/0000x.txt files
+            // and write regexed domain into it too
+            siteURLsWriter.write(domain + "\n");        
+            siteRegexWriter.write(regexed_domain + "\n");
+            
+            // next write out the urls for the domain with a tab prefixed to each
+            // into the sites/0000x/seedURLs.txt file - also write into the global seeds file
+            Set<String> urlsForDomainSet = domainsToURLsMap.get(domain);
+            Iterator<String> urlIterator = urlsForDomainSet.iterator();
+            while(urlIterator.hasNext()) {
+            String url = urlIterator.next();
+            seedURLsWriter.write(url + "\n"); // global seedURLs file
+            siteURLsWriter.write("\t" + url + "\n"); 
+            }
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+            System.err.println("\n@@@@@@@@@ Error writing to " + siteSeedsFile + " or " + siteRegexFile);
+        }
         }
         
     } catch (IOException ioe) {
         ioe.printStackTrace();
-        System.err.println("\n@@@@@@@@@ Error writing to " + urlFilterFile);
+        System.err.println("\n@@@@@@@@@ Error writing to " + seedURLsFile + " or " + urlFilterFile);
     }