source: main/trunk/model-sites-dev/hathitrust/collect/capisco-european-pacific-encounters/java/CapiscoTopicToSynonyms.java@ 31289

import java.util.zip.*;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectOutputStream;
import java.io.OutputStreamWriter;
import java.net.Socket;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;



public class CapiscoTopicToSynonyms {
      
    private static final int BUFFER_SIZE = 4096;

    private static HashMap<String,List<String>> TopicsToSynonyms = new HashMap<String,List<String>>();

    public static void main(String[] args) {
    

    if ((args.length==0) || (args.length!=3) || (args[0].endsWith("help"))) {
        System.err.println();
        System.err.println("Usage: CapiscoTopicToSynonyms 'server URL' 'server port' 'input documents path'");
        System.err.println();
        System.exit(-1);
    }

    String serverName = args[0];
    int port = Integer.parseInt(args[1]);
    String documentspath = args[2];

    try {
        System.out.println("Connecting to " + serverName
                   + " on port " + port);
        Socket client = new Socket(serverName, port);
        System.out.println("Just connected to "
                   + client.getRemoteSocketAddress());
        BufferedReader in = new BufferedReader(new InputStreamReader(client.getInputStream()));
        BufferedWriter out= new BufferedWriter(new OutputStreamWriter(client.getOutputStream()));
        File documents = new File(documentspath);

        unzipAll(documents);
        processFiles(documents, in, out);


        serializeHashmap("topics-to-synonyms.ser");

        client.close();
        
    }
    catch(IOException e) {
        System.out.println("error: "+ e.getMessage());
        e.printStackTrace();
    }
    }
    
    private static void unzipAll(File file) throws IOException {
    
    if (file.isDirectory()) 
        {
        for (File f : file.listFiles()) 
            {
            unzipAll(f);
            }
        } 
    else 
        {
            String filename = file.getName().toLowerCase();
            if(filename.endsWith(".zip"))
            {
                ZipInputStream zipIn = new ZipInputStream(new FileInputStream(file.getAbsolutePath()));
                ZipEntry entry = zipIn.getNextEntry();
                // iterates over entries in the zip file
                while (entry != null) 
                {
                    String filePath = file.getParent() + File.separator + entry.getName();
                    if (!entry.isDirectory()) 
                    {
                        // if the entry is a file, extracts it
                        extractFile(zipIn, filePath);
                    } 
                    else 
                    {
                        // if the entry is a directory, make the directory
                        File dir = new File(filePath);
                        dir.mkdir();
                    }
                    zipIn.closeEntry();
                    entry = zipIn.getNextEntry();
                }
                zipIn.close();
            }
        }
        
    
    }
    
    static String[] parseTopics(String content)
    {
    String[] topics = content.split("\\|");

    return topics;
    }


    static String getArticleId(BufferedReader input) throws IOException
    {
    String article_id = "";
    String line ="";
        while(!line.equals("--message end--") && !line.equals("0--message end--"))
        {
        if (line.startsWith("> ")) {
            line = line.substring(2);
        }
        article_id += line;
        line=input.readLine();
        }

    return article_id;
    }

    static List<String> getSynonyms(BufferedReader input) throws IOException
    {
    List<String> synonyms = new ArrayList<String>();
    String line ="";
        while(!line.equals("--message end--") && !line.equals("0--message end--"))
        {
        line=input.readLine();
        String[] words = line.split("\\|");
        for (int i=1; i<words.length; i++) {
            // deliberately skip the first value, which is the number of items in the x|y|z list
            synonyms.add(words[i]);
        }
        
        }
    return synonyms;
    }

    
    static String readFile(String path, Charset encoding) 
    throws IOException 
    {
    byte[] encoded = Files.readAllBytes(Paths.get(path));
    return new String(encoded, encoding);
    }
    

    private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException {
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath));
        byte[] bytesIn = new byte[BUFFER_SIZE];
        int read = 0;
        while ((read = zipIn.read(bytesIn)) != -1) {
            bos.write(bytesIn, 0, read);
        }
        bos.close();
    }
    
    static void processFiles(File file, BufferedReader input, BufferedWriter output) throws IOException {
    
    if (!file.exists()) 
        {
        System.out.println(file + " does not exist.");
        }
    if (file.isDirectory()) 
        {
        for (File f : file.listFiles()) 
            {
            processFiles(f, input, output);
            }
        } 
    else 
        {
        String filename = file.getName().toLowerCase();
        
        if (filename.endsWith(".txt.topics"))
            {
            String content = readFile(file.getAbsolutePath(), StandardCharsets.UTF_8);
            String[] topics = parseTopics(content);

            ArrayList<String> new_topics = new ArrayList<String>();

            for (int ti=0; ti<topics.length; ti++) {

                String topic = topics[ti];

                if (topic.equals("")) { continue; }

                if (!TopicsToSynonyms.containsKey(topic)) {
                // Step 1, convert each topic to article id
                output.write("artid " + topic + "\n");
                output.flush();
                System.out.println("Topic '" + topic + "' sent for article id lookup.");
                String article_id = getArticleId(input);
                System.out.println("  '" + topic + "' received article id: " + article_id);
                
                // Step 2, look up synonyms for each article id
                output.write("synonyms " + article_id + "\n");
                output.flush();
                System.out.println("  Article id " + article_id + " send for synonym analysis.");
                List<String> synonyms = getSynonyms(input);
                System.out.println("  " + article_id + " synonyms received (" + synonyms.size() + ")");

                // Step 3, store in hashmap
                TopicsToSynonyms.put(topic,synonyms);
                
                }
                else {
                System.out.println("  ... skipping " + topic + "as it is already in the hashmap");
                }
            }
            System.out.println(file + " processed.");
            System.out.println("=====");

            } 
        else 
            {
            System.out.println("Skipped " + filename);
            }
        }
    }

    public static void serializeHashmap(String filename)
    {
    try {
        FileOutputStream fos = new FileOutputStream(filename);
        ObjectOutputStream oos = new ObjectOutputStream(fos);
        oos.writeObject(TopicsToSynonyms);
        oos.close();
        fos.close();
        System.out.printf("Serialized HashMap data saved to:" + filename);
    }
    catch(IOException ioe) {

        ioe.printStackTrace();
    }
    }
}