source: other-projects/the-macronizer/trunk/src/java/util/MacroniserLogFileProcessor.java@ 32742

package util;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

//import MySQLAccess.Tuple;


public class MacroniserLogFileProcessor {
    /** ARGUMENTS:
     *              /home/wjkw1/RESEARCH_2018-19/bash_test/loggingtest.log
     *              /home/wjkw1/comp520/MacroniserLogs/Week02/macron.log
     */

    static final String directInputRegexp = "INFO : \\[\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d:\\d\\d\\] DirectInput.doPost\\(\\)";
    static final String fileUploadRegexp = "ERROR: \\[\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d:\\d\\d\\] FileUpload.doPost\\(\\)";

    static ArrayList<MacroniserLogFileData> extractedEntries;

    public static void main(String[] args) {
        //TODO: re enable this method and add text to command line interface
        //checkArgs(args.length);
        String filename = args[0];
        //extracts using yesterdays date
        extractFromLogFile(filename);

        //TODO: REMOVE
        int count = 0;
        for (MacroniserLogFileData entry: extractedEntries
             ) {
            printMessage((++count)+": "+entry.toString());
        }
        if(exportToDB()) {
            // TODO: move the log file to processed folder

        } else {
            // TODO: move the log file to reprocess folder
        }
    }

    //Goal is to extract information from here to DB
    private static boolean exportToDB() {

        boolean success = false;
        MySQLAccess sqlAccess = new MySQLAccess();

        try {

            sqlAccess.makeConnection();

            //loop through all entries
            for (MacroniserLogFileData entry : extractedEntries
                    ) {
                //get the marked words from first entry
                ArrayList<String> markedWords = getMarkedWordsFromOutput(entry.getOutputText());
                if (markedWords != null) {
                    for (String word : markedWords) {
                        // create the tuple
                        MySQLAccess.Tuple tuple = new MySQLAccess.Tuple(word,entry.getDate(),entry.getTime());
                        //TODO: insert all into database
                        //insert all into database
                        printMessage(word);
                        printMessage(entry.getDate().toString());
                        printMessage(entry.getTime().format(DateTimeFormatter.ofPattern("HH:mm:ss")));
                        printMessage("");

                        sqlAccess.addNewEntry(tuple);
                    }
                }
            }
            success = true;

        }catch (Exception e) {
            e.printStackTrace();
            success = false;

        } finally {
            sqlAccess.closeConnection();

        }
        return success;
    }

    //returns an array list of all marked words, null if there are none
    private static ArrayList<String> getMarkedWordsFromOutput(String outputText) {
        final Pattern TAG_REGEXP = Pattern.compile("<mark>(.+?)</mark>", Pattern.DOTALL);
        final Matcher matcher = TAG_REGEXP.matcher(outputText);
        ArrayList<String> markedWords = new ArrayList<>();
        //find all values from matches
        while (matcher.find()){
            String s = matcher.group(1);
            markedWords.add(s);
        }
        return  markedWords;
    }

    //reads the log file and creates a list of data
    private static void extractFromLogFile(String filename) {
        //TODO: change the date that is used
//        LocalDate yesterday = LocalDate.now().minusDays(1L);
        LocalDate specifiedDate = LocalDate.of(2018,11,06);

        BufferedReader br = null;
        try {
            br = new BufferedReader(new FileReader(filename));
            extractedEntries = new ArrayList<>();

            boolean firstTagFound = false;
            String line = "";
            StringBuilder extractedSB = new StringBuilder();

            while ((line = br.readLine())!= null){
                if(!firstTagFound){
                    //check for tag
                    firstTagFound = doesLineMatch(line);
                    //add to list if the first match
                    if(firstTagFound) {
                        extractedSB.append(line);
                        continue;
                    }
                }else {
                    //if line is a match, remove and keep
                    if(doesLineMatch(line)){
                        //turn the string builder text into MacroniserLogFileData object
                        MacroniserLogFileData entry = transformLogFileStringToObject(extractedSB.toString());
                        //perform operations if not null, else error
                        if(entry != null) {
                            ////check if entry date is after specified date
                            if(entry.getDate().isAfter(specifiedDate)){
                                //stop processing
                                break;
                            } else if (entry.getDate().isEqual(specifiedDate)){
                                extractedEntries.add(entry);
                            }
                            //remove old content and keep new tag
                            extractedSB.setLength(0);
                            extractedSB.append(line);
                        } else{
                            printErrorMsg("Parsing of entry in log file found an error, continuing on next lines...");
                            extractedSB.setLength(0);
                            extractedSB.append(line);
                        }

                    }else {
                        //keep the line
                        extractedSB.append(line);
                    }
                }
            }

        } catch (Exception ex) {
            ex.printStackTrace();
        } finally {
            if(br != null){
                try {
                    br.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

    }

    //Changes the string into a MacorniserLogFileData object
    private static MacroniserLogFileData transformLogFileStringToObject(String extractedString) {
        MacroniserLogFileData entry;
        LocalDate entryDate;
        LocalTime entryTime;
        String entryInput, entryOutput;

        //Get the input output portion of the string
        String input_output = "";
        if(extractedString.charAt(0)=='I'){

            //Get the date and time of entry
            int INDEX_STARTOF_DATE = 8, INDEX_ENDOF_DATE = 18, INDEX_STARTOF_TIME = 19, INDEX_ENDOF_TIME = 27;

            entryDate = LocalDate.parse(extractedString.substring(INDEX_STARTOF_DATE, INDEX_ENDOF_DATE));

            entryTime = LocalTime.parse(extractedString.substring(INDEX_STARTOF_TIME, INDEX_ENDOF_TIME));

            //Get the input and output
            input_output = extractedString.replaceAll(directInputRegexp, "").trim();

            int INDEX_STARTOF_INPUT = 6, INDEX_ENDOF_INPUT = getEndofInputIndex(input_output),
                    INDEX_STARTOF_OUTPUT = INDEX_ENDOF_INPUT + 8, INDEX_ENDOF_OUTPUT = input_output.length();
            entryInput = input_output.substring(INDEX_STARTOF_INPUT, INDEX_ENDOF_INPUT);
            entryOutput = input_output.substring(INDEX_STARTOF_OUTPUT, INDEX_ENDOF_OUTPUT);

            entry = new MacroniserLogFileData(entryDate, entryTime, entryInput, entryOutput);
            return entry;

        } else if (extractedString.charAt(0)=='E') {
            input_output = extractedString.replaceAll(fileUploadRegexp, "");
            return null;
        } else {
            printErrorMsg("Unknown entry in log file.");
            return null;
        }
    }

    //gets the end of input index, protects against log file 'input' that includes the tag
    private static int getEndofInputIndex(String str) {
        ArrayList<Integer> indexes = new ArrayList<>();
        String outputTag = "Output: ";

        //loop through and find all "Output: " Strings
        int i = str.indexOf(outputTag);
        while(i >= 0){
            indexes.add(i);
            i = str.indexOf(outputTag, i+1);
        }

        //return the correct index
        if(indexes.size() >= 1){
            int middle_index = (indexes.size() / 2);
            return indexes.get(middle_index);
        } else {
            printErrorMsg("No output tag could be found, error in log file.");
            return -1;
        }
    }

    //Does line match the regexp
    private static boolean doesLineMatch(String line) {
        return (line.matches(directInputRegexp) || line.matches(fileUploadRegexp));
    }

    //checks the arguments
    private static void checkArgs(int numArgs) {
        if( numArgs != 1){
            printErrorMsg("Incorrect number of arguments given - "+numArgs);
            printUsage();
        }
    }

    //Prints a message to std output
    private static void printMessage(String msg){
        System.out.println(msg);
    }

    //Prints the error message to "std error"
    private static void printErrorMsg(String msg){
        System.err.println("ERROR: "+ msg);
    }

    //Prints the usage of this program to "std error"
    private static void printUsage(){
        System.err.println("Usage: ");
        System.err.println("\tjava LogFileExtraction <log file>");
        System.err.println("\tWhere 'log file' is the log file of the Macroniser - macron.log");
    }
}