Changeset 31334 for other-projects/hathitrust

Timestamp:

2017-01-23T18:03:51+13:00 (7 years ago)

Author:

davidb

Message:

Initial cut at rsync download

File:

• other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java (modified) (8 diffs)

other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java

@@ -27 +27 @@
 
     protected static int HASHMAP_INIT_SIZE = 13800000;
-    protected static String DATA_DIR="D:/cygwin64/home/davidb/research/code-managed/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest";
-    protected static HashMap<String,Boolean> id_check_ = null;
+    //protected static String DATA_DIR="D:/cygwin64/home/davidb/research/code-managed/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest";
+    protected static HashMap<String,String> id_check_ = null;
     
     
@@ -35 +35 @@
     }
 
+    protected static final String opt_file_ext = ".json.bz2";
+    
+    protected String full_filename_to_tail(String full_filename)
+    {
+        String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
+        return filename_tail;
+    }
+    
+    protected String filename_tail_to_id(String filename_tail)
+    {
+        String id = null;
+        if (filename_tail.endsWith(opt_file_ext)) {
+            id = filename_tail.substring(0,filename_tail.lastIndexOf(opt_file_ext));
+        }
+        else {
+            id = filename_tail;
+        }
+        
+        id = id.replaceAll("\\+", ":").replaceAll("=", "/");
+        
+        return id;
+    }
+    
     protected void storeIDs(BufferedReader br)
     {       
@@ -40 +63 @@
         String line;
         
-        String opt_file_ext = ".json.bz2";
+        
 
         try {
 
+            System.err.print("Loading hashmap: ");
             while ((line = br.readLine()) != null) {
 
-                String file_id = line.substring(line.lastIndexOf("/")+1);
-                String id = null;
-                if (file_id.endsWith(opt_file_ext)) {
-                    id = file_id.substring(0,file_id.lastIndexOf(opt_file_ext));
-                }
-                else {
-                    id = file_id;
-                }
-
-                id = id.replaceAll("\\+", ":").replaceAll("=", "/");
-
-                id_check_.put(id, true);
-
+                String full_json_filename = line;
+                String json_filename_tail = full_filename_to_tail(full_json_filename);
+                String id = filename_tail_to_id(json_filename_tail);
+                
+                id_check_.put(id, full_json_filename);
+
+                
                 if ((line_num % 100000) == 0) {
-                    System.err.println("sample id = " + id);
-                    System.err.println("Passed line: " + line_num);
+                    //System.err.println("sample id = " + id);
+                    //System.err.println("Passed line: " + line_num);
+                    System.err.print(".");
                 }
                 line_num++;
 
             }
+            System.err.println(" => done.");
         }
         catch (Exception e) {
@@ -79 +99 @@
         
         if (id_check_ == null) {
-            id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
+            id_check_ = new HashMap<String,String>(HASHMAP_INIT_SIZE);
             
             String htrc_list_file = "htrc-ef-all-files.txt";
@@ -97 +117 @@
             }
         }
-        
-        
-         
-
-         
-    }
-
+    }
+
+    protected void doRsyncDownload(String full_json_filename)
+    {
+        String json_filename_tail = full_filename_to_tail(full_json_filename);
+        //String cmd = "rsync -av data.analytics.hathitrust.org::features/" + full_json_filename + ".";
+        
+        
+        Runtime rt = Runtime.getRuntime();
+        String[] command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
+        
+        
+        try {
+            Process proc = rt.exec(command);
+
+            /*
+            BufferedReader stdInput = new BufferedReader(new 
+                    InputStreamReader(proc.getInputStream()));
+
+            BufferedReader stdError = new BufferedReader(new 
+                    InputStreamReader(proc.getErrorStream()));
+
+            // read the output from the command
+            System.out.println("Here is the standard output of the command:\n");
+            String s = null;
+            while ((s = stdInput.readLine()) != null) {
+                System.out.println(s);
+            }
+
+            // read any errors from the attempted command
+            System.out.println("Here is the standard error of the command (if any):\n");
+            while ((s = stdError.readLine()) != null) {
+                System.out.println(s);
+            }
+            */
+
+            proc.waitFor();
+            System.err.println("*** Rsync finished");
+
+              //System.out.println("Done.");
+              
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+    }
     /**
      * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
@@ -112 +172 @@
         String cgi_ids = request.getParameter("ids");
         String cgi_id = request.getParameter("id");
+        String cgi_download_id = request.getParameter("download-id");
         
         if (cgi_ids != null) {
@@ -124 +185 @@
                 String id = ids[i];
                 
-                Boolean exists = id_check_.get(id);
-                boolean status = (exists != null) ? exists : false;
+                String full_json_filename = id_check_.get(id);
+                boolean status = (full_json_filename != null);
                 
                 if (i>0) {
@@ -139 +200 @@
             
             String id = cgi_id;
-            Boolean exists = id_check_.get(id);
-            boolean status = (exists != null) ? exists : false;
+            String full_json_filename = id_check_.get(id);
+            boolean status = (full_json_filename != null);
             pw.append("{'" + id + "':" + status + "}");
+        }
+        else if (cgi_download_id != null) {
+            String download_id = cgi_download_id;
+            String full_json_filename = id_check_.get(download_id);
+            boolean exists = (full_json_filename != null);
+            if (!exists) {
+                // Error
+                 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
+            }
+            else {
+                // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
+            
+                doRsyncDownload(full_json_filename);
+    
+            }
         }
         else {