source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31341

package org.hathitrust.extractedfeatures;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;

import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

/**
 * Servlet implementation class VolumeCheck
 */
@WebServlet("/VolumeCheck")
public class VolumeCheck extends HttpServlet {
    private static final long serialVersionUID = 1L;

    protected static int HASHMAP_INIT_SIZE = 13800000;
    protected static HashMap<String,Boolean> id_check_ = null;

    protected static final String file_ext = ".json.bz2";

    public VolumeCheck() 
    {}
    
    protected static String full_filename_to_tail(String full_filename)
    {
        String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
        return filename_tail;
    }
    
    protected static String filename_tail_to_id(String filename_tail)
    {
        String id = null;
        if (filename_tail.endsWith(file_ext)) {
            id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
        }
        else {
            id = filename_tail;
        }
        
        id = id.replaceAll("\\+", ":").replaceAll("=", "/");
        
        return id;
    }
    
    protected static String id_to_pairtree_filename(String id) {
        // Example :-
        //   id: miun.adx6300.0001.001 
        //   pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
        
        // 1. Map 'difficult' chars: 
        //   . => ,
        //   : => +
        //   / => =
        
        // 2. Process resulting string:
        //   split on first dot
        //   add "pairtree_root"
        //   then split everything else 2 chars at a time
        
        // 3. Finally add in the (safely transformed) id:
        //   append directory that is prefix-removed id (transformed to be safe)
        //   further append 'id-safe'.json.bz
        
        int id_dot_pos = id.indexOf(".");
        String id_prefix = id.substring(0,id_dot_pos);
        String id_tail = id.substring(id_dot_pos+1);
        String id_tail_safe = id_tail.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
        
        String [] pairs = id_tail_safe.split("(?<=\\G..)");
        String joined_pairs = String.join("/", pairs);

        String id_safe = id_prefix + "." + id_tail_safe;
        String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
        String filename = main_dir + "/" + id_tail_safe + "/" + id_safe + file_ext;
        
        return filename;
    }
    
    protected void storeIDs(BufferedReader br)
    {       
        long line_num = 1;
        String line;

        try {

            System.err.print("Loading hashmap: ");
            while ((line = br.readLine()) != null) {

                String full_json_filename = line;
                String json_filename_tail = full_filename_to_tail(full_json_filename);
                String id = filename_tail_to_id(json_filename_tail);
                
                id_check_.put(id, true);

                if ((line_num % 100000) == 0) {
                    //System.err.println("sample id = " + id);
                    //System.err.println("Passed line: " + line_num);
                    System.err.print(".");
                }
                line_num++;

            }
            System.err.println(" => done.");
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        
    }
    /**
     * @see Servlet#init(ServletConfig)
     */
    public void init(ServletConfig config) throws ServletException {
        super.init(config);
        
        if (id_check_ == null) {
            id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
            
            String htrc_list_file = "htrc-ef-all-files.txt";
            InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
            
            try {
                System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
                
                InputStreamReader isr = new InputStreamReader(is, "UTF-8");
                BufferedReader br = new BufferedReader(isr);

                storeIDs(br);
                br.close();
            }
            catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    protected BufferedInputStream doRsyncDownload(String full_json_filename)
    {
        String json_filename_tail = full_filename_to_tail(full_json_filename);
        
        BufferedInputStream bis = null;
        
        Runtime runtime = Runtime.getRuntime();
        String[] rsync_command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
        
        try {
            Process proc = runtime.exec(rsync_command);
            proc.waitFor();
            //System.err.println("*** Rsync finished");
        
            FileInputStream fis = new FileInputStream(json_filename_tail);
            bis = new BufferedInputStream(fis);
              
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        
        return bis;
        
    }
    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        
        
        String cgi_ids = request.getParameter("ids");
        String cgi_id = request.getParameter("id");
        String cgi_download_id = request.getParameter("download-id");
        
        if (cgi_ids != null) {
            response.setContentType("application/json");
            PrintWriter pw = response.getWriter();
            
            String[] ids = cgi_ids.split(",");
            int ids_len = ids.length;
            
            pw.append("{");
            
            for (int i=0; i<ids_len; i++) {
                String id = ids[i];
                
                boolean exists = id_check_.get(id);
                
                if (i>0) {
                    pw.append(",");
                }
                pw.append("\"" + id + "\":" + exists );
            }
            pw.append("}");
            
        }
        else if (cgi_id != null) {
            response.setContentType("application/json");
            PrintWriter pw = response.getWriter();
            
            String id = cgi_id;
            boolean exists = id_check_.get(id);
            pw.append("{'" + id + "':" + exists + "}");
        }
        else if (cgi_download_id != null) {
            String download_id = cgi_download_id;
            boolean exists = id_check_.get(download_id);
            if (!exists) {
                // Error
                 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
            }
            else {
                // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
                String full_json_filename = id_to_pairtree_filename(download_id);
                
                BufferedInputStream bis = doRsyncDownload(full_json_filename);

                if (bis == null) {
                    response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Rsync failed");
                }
                else {
                    String json_filename_tail = full_filename_to_tail(full_json_filename);
                
                    response.setContentType("application/x-bzip2");
                    response.setHeader("Content-Disposition", "attachment; filename=\"" + json_filename_tail + "\"");

                    OutputStream os=response.getOutputStream();
                    BufferedOutputStream bos = new BufferedOutputStream(os);
                    
                    byte[] buf = new byte[1024];
                    
                    while (true) {
                        int num_bytes = bis.read(buf);
                        if (num_bytes == -1) { 
                            break;
                        }
                        bos.write(buf,0,num_bytes);
                        //total_num_bytes += num_bytes;
                    }
                                        
                    bis.close();
                    bos.close();
                }
            }
        }
        else {
            PrintWriter pw = response.getWriter();
            
            pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
                        
        }
        //pw.close();
        
    }

    /**
     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        doGet(request, response);
    }

}