source: main/trunk/model-sites-dev/hathitrust/collect/capisco-european-pacific-encounters/java/CapiscoTextToTopics.java@ 31289

import java.util.zip.*;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.Socket;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;


public class CapiscoTextToTopics {
      
    private static final int BUFFER_SIZE = 4096;

    public static void main(String[] args) {
    

    if ((args.length==0) || (args.length!=3) || (args[0].endsWith("help"))) {
        System.err.println();
        System.err.println("Usage: CapiscoTextToTopic 'server URL' 'server port' 'input documents path'");
        System.err.println();
        System.exit(-1);
    }

    String serverName = args[0];
    int port = Integer.parseInt(args[1]);
    String documentspath = args[2];

    try {
        System.out.println("Connecting to " + serverName
                   + " on port " + port);
        Socket client = new Socket(serverName, port);
        System.out.println("Just connected to "
                   + client.getRemoteSocketAddress());
        BufferedReader in = new BufferedReader(new InputStreamReader(client.getInputStream()));
        BufferedWriter out= new BufferedWriter(new OutputStreamWriter(client.getOutputStream()));
        File documents = new File(documentspath);

        unzipAll(documents);
        processFiles(documents, in, out);

        client.close();
        
    }
    catch(IOException e) {
        System.out.println("error: "+ e.getMessage());
        e.printStackTrace();
    }
    }
    
    private static void unzipAll(File file) throws IOException {
    
    if (file.isDirectory()) 
        {
        for (File f : file.listFiles()) 
            {
            unzipAll(f);
            }
        } 
    else 
        {
            String filename = file.getName().toLowerCase();
            if(filename.endsWith(".zip"))
            {
                ZipInputStream zipIn = new ZipInputStream(new FileInputStream(file.getAbsolutePath()));
                ZipEntry entry = zipIn.getNextEntry();
                // iterates over entries in the zip file
                while (entry != null) 
                {
                    String filePath = file.getParent() + File.separator + entry.getName();
                    if (!entry.isDirectory()) 
                    {
                        // if the entry is a file, extracts it
                        extractFile(zipIn, filePath);
                    } 
                    else 
                    {
                        // if the entry is a directory, make the directory
                        File dir = new File(filePath);
                        dir.mkdir();
                    }
                    zipIn.closeEntry();
                    entry = zipIn.getNextEntry();
                }
                zipIn.close();
            }
        }
        
    
    }
    

    static List<String> getTopics(BufferedReader input) throws IOException
    {
    List<String> topics = new LinkedList<String>();
    String line ="";
        while(!line.equals("--message end--") && !line.equals("0--message end--"))
        {
        line=input.readLine();
        String[] words = line.split("\\|");
        if(words.length>1)
            {
            topics.add(words[1]);
            }
        }
    return topics;
    }
    
    static String readFile(String path, Charset encoding) 
    throws IOException 
    {
    byte[] encoded = Files.readAllBytes(Paths.get(path));
    return new String(encoded, encoding);
    }
    

    private static void extractFile(ZipInputStream zipIn, String filePath) throws IOException {
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath));
        byte[] bytesIn = new byte[BUFFER_SIZE];
        int read = 0;
        while ((read = zipIn.read(bytesIn)) != -1) {
            bos.write(bytesIn, 0, read);
        }
        bos.close();
    }
    
    static void processFiles(File file, BufferedReader input, BufferedWriter output) throws IOException {
    
    if (!file.exists()) 
        {
        System.out.println(file + " does not exist.");
        }
    if (file.isDirectory()) 
        {
        for (File f : file.listFiles()) 
            {
            processFiles(f, input, output);
            }
        } 
    else 
        {
        String filename = file.getName().toLowerCase();
        
        if (filename.endsWith(".htm") || filename.endsWith(".html") || filename.endsWith(".txt")) 
            {
            List<String> WMtopics;
            String topicsfilepath = file.getAbsolutePath()+".topics";
            File topicsfile = new File(topicsfilepath);
            topicsfile.createNewFile(); 
            FileOutputStream fop = null;
            fop = new FileOutputStream(topicsfile);
            
            String content = readFile(file.getAbsolutePath(), StandardCharsets.UTF_8);
            output.write("topics xxxxxx\n ");
            output.write(content + "\n");
            output.write("xxxxxx \n");
            output.flush();
            System.out.println(file.getAbsolutePath() + " send for analysis.");
            WMtopics = getTopics(input);
            System.out.println(file.getAbsolutePath() + " has concepts received.");
            for (int i = 0; i < WMtopics.size(); i++)
                {
                byte[] contentInBytes = (WMtopics.get(i)+"|").getBytes();
                fop.write(contentInBytes);
                fop.flush();
                }
            fop.close();
            System.out.println(file.getAbsolutePath() + " processed.");

            } 
        else 
            {
            System.out.println("Skipped " + filename);
            }
        }
    }
    
}