source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31335

package org.hathitrust.extractedfeatures;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;

import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

/**
 * Servlet implementation class VolumeCheck
 */
@WebServlet("/VolumeCheck")
public class VolumeCheck extends HttpServlet {
    private static final long serialVersionUID = 1L;

    protected static int HASHMAP_INIT_SIZE = 13800000;
    protected static HashMap<String,Boolean> id_check_ = null;
    
    public VolumeCheck() {
        
    }

    protected static final String file_ext = ".json.bz2";
    
    protected static String full_filename_to_tail(String full_filename)
    {
        String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
        return filename_tail;
    }
    
    protected static String filename_tail_to_id(String filename_tail)
    {
        String id = null;
        if (filename_tail.endsWith(file_ext)) {
            id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
        }
        else {
            id = filename_tail;
        }
        
        id = id.replaceAll("\\+", ":").replaceAll("=", "/");
        
        return id;
    }
    
    protected static String id_to_pairtree_filename(String id) {
        // Example :-
        //   id: miun.adx6300.0001.001 
        //   pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
        
        // 1. Map 'difficult' chars: 
        //   . => ,
        //   : => +
        //   / => =
        
        // 2. Process resulting string:
        //   split on first dot
        //   add "pairtree_root"
        //   then split everything else 2 chars at a time
        
        // 3. Finally add in the (safely transformed) id:
        //   append directory that is 'id'
        //   further append 'id'.json.bz
        
        
        String id_safe = id.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
        
        int id_dot_pos = id_safe.indexOf(".");
        String id_prefix = id_safe.substring(0,id_dot_pos);
        String id_tail = id_safe.substring(id_dot_pos+1);
        
        String [] pairs = id_tail.split("(?<=\\G..)");
        String joined_pairs = String.join("/", pairs);

        String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
        String filename = main_dir + "/" + id_safe + "/" + id_safe + file_ext;
        
        return filename;
    }
    
    protected void storeIDs(BufferedReader br)
    {       
        long line_num = 1;
        String line;

        try {

            System.err.print("Loading hashmap: ");
            while ((line = br.readLine()) != null) {

                String full_json_filename = line;
                String json_filename_tail = full_filename_to_tail(full_json_filename);
                String id = filename_tail_to_id(json_filename_tail);
                
                id_check_.put(id, true);

                if ((line_num % 100000) == 0) {
                    //System.err.println("sample id = " + id);
                    //System.err.println("Passed line: " + line_num);
                    System.err.print(".");
                }
                line_num++;

            }
            System.err.println(" => done.");
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        
    }
    /**
     * @see Servlet#init(ServletConfig)
     */
    public void init(ServletConfig config) throws ServletException {
        super.init(config);
        
        if (id_check_ == null) {
            id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
            
            String htrc_list_file = "htrc-ef-all-files.txt";
            InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
            
            try {
                System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
                
                InputStreamReader isr = new InputStreamReader(is, "UTF-8");
                BufferedReader br = new BufferedReader(isr);

                storeIDs(br);
                br.close();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    protected void doRsyncDownload(String full_json_filename)
    {
        String json_filename_tail = full_filename_to_tail(full_json_filename);
        //String cmd = "rsync -av data.analytics.hathitrust.org::features/" + full_json_filename + ".";
        
        
        Runtime rt = Runtime.getRuntime();
        String[] command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
        
        
        try {
            Process proc = rt.exec(command);

            /*
            BufferedReader stdInput = new BufferedReader(new 
                    InputStreamReader(proc.getInputStream()));

            BufferedReader stdError = new BufferedReader(new 
                    InputStreamReader(proc.getErrorStream()));

            // read the output from the command
            System.out.println("Here is the standard output of the command:\n");
            String s = null;
            while ((s = stdInput.readLine()) != null) {
                System.out.println(s);
            }

            // read any errors from the attempted command
            System.out.println("Here is the standard error of the command (if any):\n");
            while ((s = stdError.readLine()) != null) {
                System.out.println(s);
            }
            */

            proc.waitFor();
            System.err.println("*** Rsync finished");

              //System.out.println("Done.");
              
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        
    }
    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        PrintWriter pw = response.getWriter();
        
        String cgi_ids = request.getParameter("ids");
        String cgi_id = request.getParameter("id");
        String cgi_download_id = request.getParameter("download-id");
        
        if (cgi_ids != null) {
            response.setContentType("application/json");
    
            String[] ids = cgi_ids.split(",");
            int ids_len = ids.length;
            
            pw.append("{");
            
            for (int i=0; i<ids_len; i++) {
                String id = ids[i];
                
                boolean exists = id_check_.get(id);
                
                if (i>0) {
                    pw.append(",");
                }
                pw.append("\"" + id + "\":" + exists );
            }
            pw.append("}");
            
        }
        else if (cgi_id != null) {
            response.setContentType("application/json");
            
            String id = cgi_id;
            boolean exists = id_check_.get(id);
            pw.append("{'" + id + "':" + exists + "}");
        }
        else if (cgi_download_id != null) {
            String download_id = cgi_download_id;
            boolean exists = id_check_.get(download_id);
            if (!exists) {
                // Error
                 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
            }
            else {
                // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
                String full_json_filename = id_to_pairtree_filename(download_id);
                
                doRsyncDownload(full_json_filename);
    
            }
        }
        else {

            pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
                        
        }
        //pw.close();
        
    }

    /**
     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        doGet(request, response);
    }

}