Changeset 33946 for other-projects/maori-lang-detection

Timestamp:

2020-02-18T21:58:42+13:00 (4 years ago)

Author:

ak19

Message:

1. New function to handle user input assigning the newly introduced 4th column in the random samples file. In many ways similar to the function handling the 3rd column, so some code refactoring may be useful. 2. Changed the function handling the 3rd column to leave any 4th column values untouched by writing them out with whatever values they're found.

File:

• other-projects/maori-lang-detection/src/org/greenstone/atea/ManualURLInspection.java (modified) (7 diffs)

other-projects/maori-lang-detection/src/org/greenstone/atea/ManualURLInspection.java

@@ -45 +45 @@
     public static final int COUNTRY_CODE_COLUMN = 1;
     public static final int IS_REALLY_IN_MRI_COLUMN = 2;
+    public static final int QUALITY_LEVEL_COLUMN = 3;
+
+
+    /** Possible values for the Quality Level column of the csv file */
+    public static final String NAV = "NAV";
+    public static final String LITTLE_TEXT = "LITTLE_TEXT";
+    public static final String MIXED_TEXT = "MIXED_TEXT";
+    public static final String SIGNIFICANTLY_MAORI = "SIGNIFICANTLY_MAORI";
+    public static final String MAORI_PARAGRAPHS = "MAORI_PARAGRAPHS";
+    public static final String WORDS = "WORDS"; // words or titles, not full sentences
+    public static final String OTHER_LANGUAGES = "OTHER_LANGUAGES";
+    public static final String POEMS_OR_SONGS = "POEMS_OR_SONGS";
+    public static final String SINGLE_MRI_SENTENCE = "SINGLE_MRI_SENTENCE";
 
     
@@ -119 +132 @@
         String countryCode = csvRecord.get(COUNTRY_CODE_COLUMN);
         String isReallyInMRI = "";
+        String qualityLevel = null;
+        
         //String isReallyInMRI = csvRecord.get(IS_REALLY_IN_MRI_COLUMN);
         //if(!isReallyInMRI.equals("")) {
@@ -125 +140 @@
         }
         
+        if(csvRecord.isSet(QUALITY_LEVEL_COLUMN)) {
+            qualityLevel = csvRecord.get(QUALITY_LEVEL_COLUMN);
+        }
+        
         if(terminate || (!isReallyInMRI.equals("") && !isReallyInMRI.equals("?"))) {
             // if(terminate) on Ctrl-D, don't stop processing csv records
             // Instead, copy remaining records of input csv file into output csv file
             isReallyInMRI = isReallyInMRI.toUpperCase();
-            csvWriter.printRecord(url, countryCode, isReallyInMRI);
+            if(qualityLevel == null) {
+            csvWriter.printRecord(url, countryCode, isReallyInMRI);
+            } else {
+            csvWriter.printRecord(url, countryCode, isReallyInMRI, qualityLevel);
+            }
             csvWriter.flush();
             logger.info("Got record " + recordCount + ": " + url + " - " + countryCode
-                + " - " + isReallyInMRI);
+                + " - " + isReallyInMRI + " - " + qualityLevel);
         }
         else {
@@ -141 +164 @@
             System.err.println(String.format("FULL-TEXT for record %d:\n%s\n", recordCount, fulltext));
             
-            //logger.info("Got record " + recordCount + ": " + url + " - " + countryCode);
+            //logger.info("Got record " + recordCount + ": " + url + " - " + countryCode + " - " + qualityLevel);
             
             // Read Input until Ctrl-D: read System.In as bufferedReader
@@ -188 +211 @@
             }
             
-            // save the record
-            csvWriter.printRecord(url, countryCode, isReallyInMRI);
+            // Save the CSV record - even if quality level is null
+            // Because we don't want to lose the line that used to exist in the file
+            if(qualityLevel == null) {
+            csvWriter.printRecord(url, countryCode, isReallyInMRI);
+            } else {
+            csvWriter.printRecord(url, countryCode, isReallyInMRI, qualityLevel);
+            }
             csvWriter.flush();
             
-            if(isReallyInMRI == null) {
+            if(isReallyInMRI == null) { // if sys.in readLine() was terminated with Ctrl-D
             terminate = true;
             System.out.println("User entered Ctrl-D (Lin)/Ctrl-Z (Win) - terminating.");
@@ -214 +242 @@
     }
     
-    
+    /**
+     * Similar to processCSV() above, but for entering the page quality level of each web page
+     * This goes into the QUALITY_LEVEL_COLUMN column of the csv file.
+     * Web pages from some web sites commonly recurring in the csv input file tend to be largely
+     * navigation menus, so preset to NAV. Others are known to be low quality for text resources
+     * as they only have nav menus and pictures despite these being largely in Māori,
+     * which can also go under NAV.
+     * Other web sites have little text overall whether Māori or mixed with English, nav included,
+     * (LITTLE_TEXT), or significantly mixed (MRI+ENG/...) text even if a decent amount of text
+     * (MIXED_TEXT). Some sites may largely have standalone words for learning (WORDS).
+     * Other than known websites that have regular content of one of the above types,
+     * the user can enter these values for rarer websites whose web pages may pop up:
+     * NAV, LITTLE_TEXT, MIXED_TEXT, WORDS, SIGNIFICANTLY_MAORI (for decent amounts of MRI text)
+     * MAORI_PARAGRAPHS (for largely continuous paras in MRI even if there are paras in other
+     * langs) and OTHER_LANGUAGES if text not in MRI but mostly in other language,
+     * POEMS_OR_SONGS for content that's largely songs or poetry.
+    */    
+    public String processCSV_QualityLevelColumn() { 
+
+    Map<String, String> predefinedDefaultsMap = new HashMap<String, String>();
+    predefinedDefaultsMap.put("tetaurawhiri.govt.nz", NAV);
+    predefinedDefaultsMap.put("tmoa.tki.org.nz", SIGNIFICANTLY_MAORI);
+    predefinedDefaultsMap.put("paekupu.co.nz", MIXED_TEXT); // html is mixed, but display is more MRI
+    predefinedDefaultsMap.put("m.biblepub.com", SIGNIFICANTLY_MAORI);
+    predefinedDefaultsMap.put("biblehub.com", SIGNIFICANTLY_MAORI);
+    predefinedDefaultsMap.put("pukoro.co.nz", WORDS);
+    predefinedDefaultsMap.put("mi.wikipedia.org", MIXED_TEXT);
+    predefinedDefaultsMap.put("mi.m.wikipedia.org", WORDS);
+    predefinedDefaultsMap.put("tkkmmokopuna.school.nz", NAV);
+    predefinedDefaultsMap.put("twtop.school.nz", NAV);
+    predefinedDefaultsMap.put("animations.tewhanake.maori.nz", MAORI_PARAGRAPHS);
+    predefinedDefaultsMap.put("csunplugged.org", SIGNIFICANTLY_MAORI);
+    predefinedDefaultsMap.put("waiata.maori.nz", POEMS_OR_SONGS);
+    
+    final String USER_PROMPT = "Enter qualityLevel value of\n\t? | (N)AV | (L)ITTLE_TEXT | (M)IXED_TEXT | (S)IGNIFICANTLY_MAORI | MAORI_(P)ARAGRAPHS"
+        + "\n\t | PO(E)MS_OR_SONGS | S(I)NGLE_MRI_SENTENCE | (W)ORDS | (O)THER_LANGUAGES\n\tfor (%d): %s - %s > ";
+        //"Enter isMRI value of Y|N|? for (" + count + "): " + url + " - " + countryCode + " > ";
+    
+    boolean terminate = false;
+    CSVParser parser = null;
+    
+    try {
+        parser = CSVParser.parse(webPageURLsCSVFile, java.nio.charset.Charset.forName("US-ASCII"), CSVFormat.RFC4180);
+    } catch(Exception e) {
+        logger.error("Failed to parse input CSV file " + Utility.getFilePath(webPageURLsCSVFile), e);
+        return "Failed";
+    }
+    
+    try (
+         CSVPrinter csvWriter = new CSVPrinter(new FileWriter(tmpOutFile), CSVFormat.DEFAULT.withQuoteMode(QuoteMode.MINIMAL));
+         ) {
+
+        int recordCount = 0;
+        for (CSVRecord csvRecord : parser) {        
+        //if(terminate) condition handled further below
+        
+        //logger.debug("Got record: " + csvRecord.toString());
+        
+        String url = csvRecord.get(URL_COLUMN);
+        if(url.equals("")) { // skip empty lines
+            continue;
+        }
+        
+        recordCount++;
+        String countryCode = csvRecord.get(COUNTRY_CODE_COLUMN);
+        String isReallyInMRI = "";
+        String qualityLevel = "";
+        
+        if(csvRecord.isSet(IS_REALLY_IN_MRI_COLUMN)) {
+            isReallyInMRI = csvRecord.get(IS_REALLY_IN_MRI_COLUMN);
+        }
+        
+        if(csvRecord.isSet(QUALITY_LEVEL_COLUMN)) {
+            qualityLevel = csvRecord.get(QUALITY_LEVEL_COLUMN);
+
+            /*
+            qualityLevel = qualityLevel.toUpperCase();
+            
+            if(qualityLevel.equals("N")) {
+            qualityLevel = NAV;
+            } else if(qualityLevel.equals("L")) {
+            qualityLevel = LITTLE_TEXT;
+            } else if(qualityLevel.equals("M")) {
+            qualityLevel = MIXED_TEXT;
+            } else if(qualityLevel.equals("P")) {
+            qualityLevel = MAORI_PARAGRAPHS;
+            } else if(qualityLevel.equals("S")) {
+            qualityLevel = SIGNIFICANTLY_MAORI;
+            } else if(qualityLevel.equals("W")) {
+            qualityLevel = WORDS;
+            } else if(qualityLevel.equals("O")) {
+            qualityLevel = OTHER_LANGUAGE;
+            } else if(qualityLevel.equals("E")) {
+            qualityLevel = POEMS_OR_SONGS;
+            } else if(qualityLevel.equals("I")) {
+            qualityLevel = SINGLE_MRI_SENTENCE;
+            }
+            // else remains at whatever was already in the file or
+            // else "" if no qualityLevel column for this record present in the file yet
+            */
+
+            // Force valid values or ""
+            qualityLevel = getFullQualityLevelNameUppercased(qualityLevel);
+        }       
+        
+        if(terminate || (!qualityLevel.equals("") && !qualityLevel.equals("?"))) {
+            // if(terminate) on Ctrl-D, don't stop processing csv records
+            // Instead, copy remaining records of input csv file into output csv file      
+            
+            csvWriter.printRecord(url, countryCode, isReallyInMRI, qualityLevel);
+            csvWriter.flush();
+            logger.info("Got record " + recordCount + ": " + url + " - " + countryCode
+                + " - " + isReallyInMRI + " - " + qualityLevel);            
+        }
+        else {
+            
+            // First, display full text for web page record with matching url
+            // so the user can look at it to decide whether it is indeed overall in MRI or not.
+            String fulltext = mongodbQueryer.displayFullTextOfPage(url);
+            System.err.println(String.format("\nFULL-TEXT for record %d:\n%s\n", recordCount, fulltext));
+            
+            //logger.info("Got record " + recordCount + ": " + url + " - " + countryCode + " - " + qualityLevel);
+            
+            // Read Input until Ctrl-D: read System.In as bufferedReader
+            // https://stackoverflow.com/questions/5837823/read-input-until-controld
+            // Ctrl-C is already taken care if, see
+            // https://coderanch.com/t/279136/java/terminated-program-Control-close-open
+            // "Whenever a process is terminated/killed(CTRL-C), the file descriptors are released. You really do not need to close the stream in such cases."
+            // So I just need to flush the csv print writer after every record is written
+            // and Ctrl-C won't lose any of the data thus far entered by the user.
+            
+            BufferedReader systemIn = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));            
+            
+            boolean done = false;           
+            
+            // Work out default if basic URLs present in defaults map
+            // If it is, use its value as default for this URL
+            String basicURL = Utility.stripProtocolAndWWWFromURL(Utility.getDomainForURL(url, false));
+            String predefQualityLevel = predefinedDefaultsMap.get(basicURL);
+            
+            System.out.println(String.format(USER_PROMPT, recordCount, url, countryCode));
+            if(predefQualityLevel != null) {
+            System.err.println("\tDefault for this domain: " + predefQualityLevel
+                       + ". Press Enter to accept >");
+            }           
+            
+            boolean previouslyQuestionMark = false;
+            String oldQualityLevel = qualityLevel;
+            
+            if(qualityLevel.equals("?")) {
+            previouslyQuestionMark = true;
+            System.err.println("\t? entered last time. Press Enter to keep >");
+            }
+            while(!done && ((qualityLevel = systemIn.readLine()) != null)) {            
+            //logger.debug("@@ Got: |" + qualityLevel + "|");
+
+            // If the user hit enter, it means they accepted
+            // - the previous value entered, if it was a ?
+            // - or want the default for the URL if any displayed
+            // - or want SIGNIFICANTLY_MAORI if no default displayed
+            if(qualityLevel.equals("")) { // User just hit enter without other chars
+                if(previouslyQuestionMark) {
+                qualityLevel = "?";
+                } else {
+                qualityLevel = (predefQualityLevel == null) ? SIGNIFICANTLY_MAORI : predefQualityLevel;
+                }
+
+                oldQualityLevel = qualityLevel;
+            }
+            else {
+                // force valid values - will return "" if invalid value
+                qualityLevel = getFullQualityLevelNameUppercased(qualityLevel);
+            }
+
+            // only if qualityLevel entered was invalid, would it now
+            // have been changed to ""
+            if(!qualityLevel.equals("")) {
+                oldQualityLevel = qualityLevel;
+                done = true;
+            } else { 
+                System.out.println("@@ UNRECOGNISED. "
+                   + String.format(USER_PROMPT, recordCount, url, countryCode));
+            }
+            }           
+            
+            // Save the CSV record - even if quality level is null
+            // Because we don't want to lose the line that used to exist in the file
+            csvWriter.printRecord(url, countryCode, isReallyInMRI, qualityLevel);
+            csvWriter.flush();
+            
+            if(qualityLevel == null) { // if sys.in readLine() was terminated with Ctrl-D
+            terminate = true;
+            System.out.println("--- Got Ctrl-D (Lin)/Ctrl-Z (Win). Terminating. ---");
+            } else {
+            System.out.println("User entered: " + oldQualityLevel);
+            
+            }           
+        }
+        }
+
+        if(terminate = true) {
+        System.out.println("User entered Ctrl-D (Lin)/Ctrl-Z (Win) - terminating.");
+        }
+        
+    } catch(Exception e) {
+        e.printStackTrace();
+        logger.error("Exception occurred when processing CSV file or writing out file:\n"
+             + Utility.getFilePath(tmpOutFile));
+        logger.error(e.getMessage(), e);
+    }
+    
+    
+    return Utility.getFilePath(tmpOutFile);
+    }
+
+    public String getFullQualityLevelNameUppercased(String qualityLevel) {
+    
+    qualityLevel = qualityLevel.toUpperCase();
+
+    if(qualityLevel.equals("N")) {
+        return NAV;
+    } else if(qualityLevel.equals("L")) {
+        return LITTLE_TEXT;
+    } else if(qualityLevel.equals("M")) {
+        return MIXED_TEXT;
+    } else if(qualityLevel.equals("S")) {
+        return SIGNIFICANTLY_MAORI;
+    } else if(qualityLevel.equals("P")) {
+        return MAORI_PARAGRAPHS;
+    } else if(qualityLevel.equals("W")) {
+        return WORDS;
+    } else if(qualityLevel.equals("O")) {
+        return OTHER_LANGUAGES;
+    } else if(qualityLevel.equals("E")) {
+        return POEMS_OR_SONGS;
+    } else if(qualityLevel.equals("I")) {       
+        return SINGLE_MRI_SENTENCE;
+    } else if(qualityLevel.equals(NAV)
+          || qualityLevel.equals(LITTLE_TEXT)
+          || qualityLevel.equals(MIXED_TEXT)
+          || qualityLevel.equals(SIGNIFICANTLY_MAORI)
+          || qualityLevel.equals(MAORI_PARAGRAPHS)
+          || qualityLevel.equals(WORDS)
+          || qualityLevel.equals(OTHER_LANGUAGES)
+          || qualityLevel.equals(POEMS_OR_SONGS)
+          || qualityLevel.equals(SINGLE_MRI_SENTENCE)) {
+        return qualityLevel;
+    }
+    return "";
+    }
     
     public static void printUsage() {
@@ -266 +543 @@
         public void run() {
             logger.info("@@@@@@@@@@@@@@@@@@@@@@@@");
-            logger.info("WARNING!!!");
-            logger.info("Got Ctrl-C. INCOMPLETE generated temp CSV file: " +
+            logger.info("WARNING: If Ctrl-C was pressed, then");
+            logger.info("\tan INCOMPLETE temp CSV file would have been generated at: " +
                 inspector.getCSVOutputFilename());
-            logger.info(String.format("Copy remaining records from input file %s into this file.",
+            logger.info(String.format("\tSo copy remaining records from input file %s into this file.",
                           Utility.getFilePath(inputFile)));
             logger.info("@@@@@@@@@@@@@@@@@@@@@@@@");
         }
         }));
-        
-        String filename = inspector.processCSV();
+
+        //String filename = inspector.processCSV();
+        String filename = inspector.processCSV_QualityLevelColumn();
+        
         
         logger.info("Generated temp CSV file: " + filename);