Changeset 33468

Timestamp:

2019-09-13T19:24:27+12:00 (5 years ago)

Author:

ak19

Message:

More meaningful to (also) write out the keep vs discard URLs into keep and discard .txt files, than just write out the WET records. That way, we have URLs to start downloading sites from (after removing duplicates/URLs of the same domain).

File:

• gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java (modified) (13 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/WETProcessor.java

@@ -21 +21 @@
  *
  * e.g.
- *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
- *    - java -cp ".:../conf" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
+ *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET
+ *    - java -cp ".:../conf:../lib/*" org.greenstone.atea.WETProcessor ../tmp/processWET /Scratch/ak19/gs3-extensions/maori-lang-detection/tmp/processedWET 2>&1 | less
  *
 */
@@ -55 +54 @@
     private static final int MIN_CONTENT_LENGTH_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.content.length.wrapped.line", "500"));
     private static final int MIN_SPACES_IN_A_WRAPPED_LINE = Integer.parseInt(configProperties.getProperty("WETprocessor.min.spaces.per.wrapped.line", "10"));
+
+    // File paths shared across WETProcessor instances
+    private static File discardFolder;
+    private static File keepFolder;
+    private static File keepURLsFile;
+    private static File discardURLsFile;
+
+    // WARC WET header lines and header line prefixes of interest
+    static final String WARC_RECORD_START = "WARC/1.0";
+    static final String WARC_INFO_HEADER = "WARC-Type: warcinfo";
+    static final String WARC_TARGET_URI_HEADER_PREFIX = "WARC-Target-URI:";
+    static final String WARC_CONTENT_LENGTH_HEADER_PREFIX = "Content-Length:";
     
     // Keep a count of all the records that all WETProcessors instantiated
@@ -76 +87 @@
     String line = null; 
     boolean readingRecord = false;
+
+    String WARCtargetURI = "";
     
     int recordCount = 0;
@@ -92 +105 @@
     fileID = fileID.substring(0, fileID.indexOf("."));
     this.WETFileID = fileID;
-    
+
+        
     // read from WETfile
-    try (BufferedReader reader = new BufferedReader(new FileReader(inFile))) {
+    try (
+         BufferedReader reader = new BufferedReader(new FileReader(inFile));         
+         BufferedWriter keepURLsWriter = new BufferedWriter(new FileWriter(keepURLsFile, true));
+         BufferedWriter discardURLsWriter = new BufferedWriter(new FileWriter(discardURLsFile, true)); // true to append
+         ) {
         
         while((line = reader.readLine()) != null) { // readLine removes newline separator
 
-        if(line.startsWith("WARC-Type: warcinfo")) {
+        if(line.startsWith(WARC_INFO_HEADER)) { // "WARC-Type: warcinfo"
             readingRecord = false;
             record = null; // drop this record, which is just an info record not actual web page's text
@@ -105 +123 @@
         }
         
-        if(line.startsWith("WARC/1.0")) { // finished previous WET record
+        if(line.startsWith(WARC_RECORD_START)) { // "WARC/1.0" means finished previous WET record
             // process any previous record
             if(record != null) {
-            processWETrecord(recordCount, contentLength, lineCount, record.toString());
+            processWETrecord(keepURLsWriter, discardURLsWriter,
+                     recordCount, contentLength, lineCount,
+                     WARCtargetURI, record.toString());
             record = null;
             contentLength = -1;
@@ -122 +142 @@
         if(readingRecord) { // append current line to current record
 
+            if(line.startsWith(WARC_TARGET_URI_HEADER_PREFIX)) { // "WARC-Target-URI:"
+            // get and store the value
+            WARCtargetURI = line.substring(WARC_TARGET_URI_HEADER_PREFIX.length()).trim();
+            }
+            
             record.append(line + "\n"); // add back (unix style) line ending
 
@@ -132 +157 @@
             lineCount++;
             }
-            else if(line.startsWith("Content-Length:")) {
-            String contentLengthStr = line.substring("Content-Length:".length()).trim();
+            else if(line.startsWith(WARC_CONTENT_LENGTH_HEADER_PREFIX)) { // "Content-Length:"
+            String contentLengthStr = line.substring(WARC_CONTENT_LENGTH_HEADER_PREFIX.length()).trim();
             contentLength = Integer.parseInt(contentLengthStr);
             lineCount = 0;
@@ -144 +169 @@
         // flush the last record. If it was a warcinfo record, record would be null here
         if(record != null) {
-        processWETrecord(recordCount, contentLength, lineCount, record.toString());
+        processWETrecord(keepURLsWriter, discardURLsWriter,
+                 recordCount, contentLength, lineCount,
+                 WARCtargetURI, record.toString());
         record = null;
         }
@@ -156 +183 @@
      * Determines if a WET record belongs in the keep or discard pile depending on if it
      * contains enough text, based on contentLength and line count of the record body.
-     * Then writes out the WET record to a uniquely named file in the keep or discard folder.
+     * Then writes out the WET record to a uniquely named file in the keep or discard folder,
+     * and writes out the WET record's URL to the keepURLs.txt file or discardURLs.txt file.
      */
-    private void processWETrecord(int recordID, int contentLength, int lineCount, String record)
+    private void processWETrecord(BufferedWriter keepURLsWriter, BufferedWriter discardURLsWriter,
+                  int recordID, int contentLength, int lineCount,
+                  String recordURI, String record)
     {
     System.err.println("WET #" + this.WETFileID + " record #" + recordID
                + " - contentLength: " + contentLength
                + " - lineCount: " + lineCount);
+    System.err.println("URI: " + recordURI);
     //System.err.println(record);
     //System.err.println("--------------------------");
@@ -168 +199 @@
     String paddedFileName = String.format("%04d.txt", recordID);
     
-    File discardFolder = new File(this.outputFolder, "discard");        
-    File keepFolder = new File(this.outputFolder, "keep");
     File parentFolder = null;
     
     if(lineCount >= MIN_LINE_COUNT && contentLength >= MIN_CONTENT_LENGTH) {
-        parentFolder = keepFolder;
+        parentFolder = WETProcessor.keepFolder;
         System.err.println("@@@KEEPING");
     } else if(contentLength >= MIN_CONTENT_LENGTH_WRAPPED_LINE) {
@@ -183 +212 @@
         // So we have at least 500 chars (possibly on a single wrapped line)
         // containing at least 10 spaces. Such a record is also worth keeping.
-        parentFolder = keepFolder;
+        parentFolder = WETProcessor.keepFolder;
         }
     }
@@ -189 +218 @@
     // if parentFolder still not set, set to discard pile folder
     if(parentFolder == null) {
-        parentFolder = discardFolder;
+        parentFolder = WETProcessor.discardFolder;
         System.err.println("@@@DISCARDING");
     }
 
+    try {
+        if (parentFolder == WETProcessor.keepFolder) {
+        keepURLsWriter.write(recordURI + "\n");
+        } else {
+        discardURLsWriter.write(recordURI + "\n");
+        }
+    } catch(Exception e) {
+        System.err.println("Unable to write URL");
+        e.printStackTrace();
+    }
+   
     System.err.println("--------------------------");
     
@@ -265 +305 @@
         return;
     }   
-    
-    File discardFolder = new File(outFolder, "discard");
-    if(!discardFolder.exists()) {
-        discardFolder.mkdir();
+
+    // static folders and files to be shared across all WETProcessor instances
+    WETProcessor.discardFolder = new File(outFolder, "discard");
+    if(!WETProcessor.discardFolder.exists()) {
+        WETProcessor.discardFolder.mkdir();
     }   
-    File keepFolder = new File(outFolder, "keep");
-    if(!keepFolder.exists()) {
-        keepFolder.mkdir();
-    }
-
+    WETProcessor.keepFolder = new File(outFolder, "keep");
+    if(!WETProcessor.keepFolder.exists()) {
+        WETProcessor.keepFolder.mkdir();
+    }
+
+    WETProcessor.keepURLsFile = new File(outFolder, "keepURLs.txt");    
+    if(WETProcessor.keepURLsFile.exists() && !WETProcessor.keepURLsFile.delete()) {
+        System.err.println("Unable to delete " + WETProcessor.keepURLsFile + ". Unable to proceed.");
+        return;
+    }
+    WETProcessor.discardURLsFile = new File(outFolder, "discardURLs.txt");
+    if(WETProcessor.discardURLsFile.exists() && !WETProcessor.discardURLsFile.delete()) {
+        System.err.println("Unable to delete " + WETProcessor.discardURLsFile + ". Unable to proceed.");
+        return;
+    }
+    
     // Will list all the warc.wet files in the input directory or else their gzipped versions
     File[] WETFiles = WETFileDir.listFiles(new WETFilenameFilter());