Changeset 31335 for other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java

Timestamp:

2017-01-23T20:37:32+13:00 (7 years ago)

Author:

davidb

Message:

Too expensive to hold pairtree filename in hashmap, so change to computing on-the-fly from 'id'

File:

• other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java (modified) (9 diffs)

other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java

@@ -27 +27 @@
 
     protected static int HASHMAP_INIT_SIZE = 13800000;
-    //protected static String DATA_DIR="D:/cygwin64/home/davidb/research/code-managed/hathitrust/wcsa/extracted-features-solr/trunk/solr-ingest";
-    protected static HashMap<String,String> id_check_ = null;
+    protected static HashMap<String,Boolean> id_check_ = null;
     
-    
     public VolumeCheck() {
         
     }
 
-    protected static final String opt_file_ext = ".json.bz2";
-    
-    protected String full_filename_to_tail(String full_filename)
+    protected static final String file_ext = ".json.bz2";
+    
+    protected static String full_filename_to_tail(String full_filename)
     {
         String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
@@ -43 +41 @@
     }
     
-    protected String filename_tail_to_id(String filename_tail)
+    protected static String filename_tail_to_id(String filename_tail)
     {
         String id = null;
-        if (filename_tail.endsWith(opt_file_ext)) {
-            id = filename_tail.substring(0,filename_tail.lastIndexOf(opt_file_ext));
+        if (filename_tail.endsWith(file_ext)) {
+            id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
         }
         else {
@@ -56 +54 @@
         
         return id;
+    }
+    
+    protected static String id_to_pairtree_filename(String id) {
+        // Example :-
+        //   id: miun.adx6300.0001.001 
+        //   pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
+        
+        // 1. Map 'difficult' chars: 
+        //   . => ,
+        //   : => +
+        //   / => =
+        
+        // 2. Process resulting string:
+        //   split on first dot
+        //   add "pairtree_root"
+        //   then split everything else 2 chars at a time
+        
+        // 3. Finally add in the (safely transformed) id:
+        //   append directory that is 'id'
+        //   further append 'id'.json.bz
+        
+        
+        String id_safe = id.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
+        
+        int id_dot_pos = id_safe.indexOf(".");
+        String id_prefix = id_safe.substring(0,id_dot_pos);
+        String id_tail = id_safe.substring(id_dot_pos+1);
+        
+        String [] pairs = id_tail.split("(?<=\\G..)");
+        String joined_pairs = String.join("/", pairs);
+
+        String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
+        String filename = main_dir + "/" + id_safe + "/" + id_safe + file_ext;
+        
+        return filename;
     }
     
@@ -62 +95 @@
         long line_num = 1;
         String line;
-        
-        
 
         try {
@@ -74 +105 @@
                 String id = filename_tail_to_id(json_filename_tail);
                 
-                id_check_.put(id, full_json_filename);
-
-                
+                id_check_.put(id, true);
+
                 if ((line_num % 100000) == 0) {
                     //System.err.println("sample id = " + id);
@@ -99 +129 @@
         
         if (id_check_ == null) {
-            id_check_ = new HashMap<String,String>(HASHMAP_INIT_SIZE);
+            id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
             
             String htrc_list_file = "htrc-ef-all-files.txt";
@@ -185 +215 @@
                 String id = ids[i];
                 
-                String full_json_filename = id_check_.get(id);
-                boolean status = (full_json_filename != null);
+                boolean exists = id_check_.get(id);
                 
                 if (i>0) {
                     pw.append(",");
                 }
-                pw.append("\"" + id + "\":" + status );
+                pw.append("\"" + id + "\":" + exists );
             }
             pw.append("}");
@@ -200 +229 @@
             
             String id = cgi_id;
-            String full_json_filename = id_check_.get(id);
-            boolean status = (full_json_filename != null);
-            pw.append("{'" + id + "':" + status + "}");
+            boolean exists = id_check_.get(id);
+            pw.append("{'" + id + "':" + exists + "}");
         }
         else if (cgi_download_id != null) {
             String download_id = cgi_download_id;
-            String full_json_filename = id_check_.get(download_id);
-            boolean exists = (full_json_filename != null);
+            boolean exists = id_check_.get(download_id);
             if (!exists) {
                 // Error
@@ -214 +241 @@
             else {
                 // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
-            
+                String full_json_filename = id_to_pairtree_filename(download_id);
+                
                 doRsyncDownload(full_json_filename);