Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33578)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/NutchTextDumpProcessor.java	(revision 33579)
@@ -36,9 +36,9 @@
  * TO COMPILE:
  *    maori-lang-detection/src$
- *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/*" org/greenstone/atea/NutchTextDumpProcessor.java
+ *       javac -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor.java
  *
  * TO RUN:
  *    maori-lang-detection/src$
- *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor
+ *       java -cp ".:../conf:../lib/*:$OPENNLP_HOME/lib/opennlp-tools-1.9.1.jar" org/greenstone/atea/NutchTextDumpProcessor ../crawled
  *
 */
@@ -67,11 +67,27 @@
 	     ) {
 
+	    boolean readingText = false;
+	    
 	    while((line = reader.readLine()) != null) { // readLine removes newline separator
 		line = line.trim();
-		// an empty line marks the end of a page in nutch's text dump of a site
-		if(!line.equals("")) { 
+		// iff outside of a page's body text, then an empty line marks the end of a page
+		// in nutch's text dump of a site.
+		// But note, there can be an empty line (or more?) between the start and end
+		// markers of a page's text, though.
+
+		if(!readingText && line.equals("")) { 
 		    pageDump.append(line);
 		    pageDump.append("\n");
-		} else {
+
+		    
+		    // START DEBUG
+		    debug("__________________________________________");
+		    debug("@@@ Found page entry: ");
+		    debug("__________________________________________");
+		    debug(pageDump.toString());
+		    debug("------------------------------------------");
+		    // END DEBUG
+		    
+		    
 		    TextDumpPage page = new TextDumpPage(siteID, pageDump.toString());
 		    // parses the fields and body text of a webpage in nutch's txt dump of entire site
@@ -80,6 +96,18 @@
 		    pages.add(page);
 		    pageDump = null;
+		    
 		    pageDump = new StringBuilder();
 		}
+		else if (!line.equals("")) { // empty line
+		    if(line.equals(TextDumpPage.TEXT_START_MARKER)) {
+			readingText = true;
+		    }
+		    if(line.equals(TextDumpPage.TEXT_END_MARKER)) {
+			readingText = false;
+		    }
+		    pageDump.append(line);
+		    pageDump.append("\n");
+		}
+		// can throw away any newlines between text start and end markers.
 	    }
 		
@@ -171,6 +199,6 @@
 		else {
 		    String siteID = siteDir.getName();
-		    NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);
-		    
+		    debug("Found siteID: " + siteID);		    
+		    NutchTextDumpProcessor nutchTxtDump = new NutchTextDumpProcessor(mriTxtDetector, siteID, txtDumpFile);		    
 		}
 		
Index: /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java
===================================================================
--- /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33578)
+++ /gs3-extensions/maori-lang-detection/src/org/greenstone/atea/TextDumpPage.java	(revision 33579)
@@ -27,10 +27,24 @@
 	    // first line always has a "key:" somewhere after the pageURL
 	    int endIndex = line.indexOf("key:");
-	    String pageURL = line.substring(endIndex);
+
+	    //String pageURL = line.substring(0, endIndex);
+	    String pageURL = (endIndex == -1) ? line : line.substring(0, endIndex);	    
+	    
 	    
 	    tuples.put("pageURL", pageURL.trim());
 
-	    String key = line.substring(endIndex);
-	    tuples.put("key", key.trim());
+	    if(endIndex != -1) {
+		String key = line.substring(endIndex);
+		tuples.put("key", key.trim());
+	    } else {
+		debug("@@@@ no key for pageURL: " + pageURL);
+	    }
+
+	    if(pageURL.contains(TEXT_END_MARKER)) {
+		debug("@@@@ TEXT_END_MARKER assigned to pageURL for page: ");
+		debug("+++++++++");
+		debug(unparsedPageDump);
+		debug("+++++++++");
+	    }
 
 	    boolean readingPageText = false;
@@ -77,8 +91,21 @@
 		tuples.put("pageText", "");
 	    }
+
 	    
 	} catch (IOException ioe) {
 	    error("@@@@@@@@@ Error reading in txtdump of a page.", ioe);
 	}
+
+	/*
+	// START DEBUG
+	debug("__________________________________________");
+	for(Map.Entry<String, String> entry : tuples.entrySet()) {
+	    String key = entry.getKey();
+	    String value = entry.getValue();	    
+	    debug(key + " - " + value);
+	}
+	debug("__________________________________________");
+	// END DEBUG
+	*/
     }
 
