Changeset 33579

Timestamp:

2019-10-17T21:05:21+13:00 (5 years ago)

Author:

ak19

Message:

Debugging. Solved one problem.

Location:

gs3-extensions/maori-lang-detection/src/org/greenstone/atea

Files:

• NutchTextDumpProcessor.java (modified) (4 diffs)
• TextDumpPage.java (modified) (2 diffs)

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java

@@ -36 +36 @@
  * TO COMPILE:
  *    maori-lang-detection/src$
- *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/*" org/greenstone/atea/NutchTextDumpProcessor.java
+ *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor.java
  *
  * TO RUN:
  *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled
  *
 */
@@ -67 +67 @@
          ) {
 
+        boolean readingText = false;
+        
         while((line = reader.readLine()) != null) { // readLine removes newline separator
         line = line.trim();
-        // an empty line marks the end of a page in nutch's text dump of a site
-        if(!line.equals("")) { 
+        // iff outside of a page's body text, then an empty line marks the end of a page
+        // in nutch's text dump of a site.
+        // But note, there can be an empty line (or more?) between the start and end
+        // markers of a page's text, though.
+
+        if(!readingText && line.equals("")) { 
             pageDump.append(line);
             pageDump.append("\n");
-        } else {
+
+            
+            // START DEBUG
+            debug("__________________________________________");
+            debug("@@@ Found page entry: ");
+            debug("__________________________________________");
+            debug(pageDump.toString());
+            debug("------------------------------------------");
+            // END DEBUG
+            
+            
             TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
             // parses the fields and body text of a webpage in nutch's txt dump of entire site
@@ -80 +96 @@
             pages.add(page);
             pageDump = null;
+            
             pageDump = new StringBuilder();
         }
+        else if (!line.equals("")) { // empty line
+            if(line.equals(TextDumpPage.TEXT_START_MARKER)) {
+            readingText = true;
+            }
+            if(line.equals(TextDumpPage.TEXT_END_MARKER)) {
+            readingText = false;
+            }
+            pageDump.append(line);
+            pageDump.append("\n");
+        }
+        // can throw away any newlines between text start and end markers.
         }
         
@@ -171 +199 @@
         else {
             String siteID = siteDir.getName();
-            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
-            
+            debug("Found siteID: " + siteID);           
+            NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);          
         }
         

gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java

@@ -27 +27 @@
         // first line always has a "key:" somewhere after the pageURL
         int endIndex = line.indexOf("key:");
-        String pageURL = line.substring(endIndex);
+
+        //String pageURL = line.substring(0, endIndex);
+        String pageURL = (endIndex == -1) ? line : line.substring(0, endIndex);     
+        
         
         tuples.put("pageURL", pageURL.trim());
 
-        String key = line.substring(endIndex);
-        tuples.put("key", key.trim());
+        if(endIndex != -1) {
+        String key = line.substring(endIndex);
+        tuples.put("key", key.trim());
+        } else {
+        debug("@@@@ no key for pageURL: " + pageURL);
+        }
+
+        if(pageURL.contains(TEXT_END_MARKER)) {
+        debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
+        debug("+++++++++");
+        debug(unparsedPageDump);
+        debug("+++++++++");
+        }
 
         boolean readingPageText = false;
@@ -77 +91 @@
         tuples.put("pageText", "");
         }
+
         
     } catch (IOException ioe) {
         error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
     }
+
+    /*
+    // START DEBUG
+    debug("__________________________________________");
+    for(Map.Entry<String, String> entry : tuples.entrySet()) {
+        String key = entry.getKey();
+        String value = entry.getValue();        
+        debug(key + " - " + value);
+    }
+    debug("__________________________________________");
+    // END DEBUG
+    */
     }