source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31347

Last change on this file since 31347 was 31347, checked in by davidb, 7 years ago

First stage of developing HT collection to HTRC workset. Code to allow this server to proxy the download function of the HT server
  • Property svn:executable set to *
File size: 10.4 KB
Line 
1package org.hathitrust.extractedfeatures;
2
3import java.io.BufferedInputStream;
4import java.io.BufferedOutputStream;
5import java.io.BufferedReader;
6import java.io.DataOutputStream;
7import java.io.FileInputStream;
8import java.io.FileReader;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.InputStreamReader;
12import java.io.OutputStream;
13import java.io.PrintWriter;
14import java.io.UnsupportedEncodingException;
15import java.net.HttpURLConnection;
16import java.net.URL;
17import java.nio.charset.StandardCharsets;
18import java.nio.file.Files;
19import java.nio.file.Path;
20import java.nio.file.Paths;
21import java.util.ArrayList;
22import java.util.HashMap;
23
24import javax.servlet.ServletConfig;
25import javax.servlet.ServletException;
26import javax.servlet.annotation.WebServlet;
27import javax.servlet.http.HttpServlet;
28import javax.servlet.http.HttpServletRequest;
29import javax.servlet.http.HttpServletResponse;
30
31
32/**
33 * Servlet implementation class VolumeCheck
34 */
35@WebServlet("/VolumeCheck")
36public class VolumeCheck extends HttpServlet {
37 private static final long serialVersionUID = 1L;
38
39 protected static int HASHMAP_INIT_SIZE = 13800000;
40 protected static HashMap<String,Boolean> id_check_ = null;
41
42 protected static final String file_ext = ".json.bz2";
43 protected static final String ht_col_url = "https://babel.hathitrust.org/cgi/mb";
44
45 public VolumeCheck()
46 {}
47
48 protected static String full_filename_to_tail(String full_filename)
49 {
50 String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
51 return filename_tail;
52 }
53
54 protected static String filename_tail_to_id(String filename_tail)
55 {
56 String id = null;
57 if (filename_tail.endsWith(file_ext)) {
58 id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
59 }
60 else {
61 id = filename_tail;
62 }
63
64 id = id.replaceAll("\\+", ":").replaceAll("=", "/");
65
66 return id;
67 }
68
69 protected static String id_to_pairtree_filename(String id) {
70 // Example :-
71 // id: miun.adx6300.0001.001
72 // pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
73
74 // 1. Map 'difficult' chars:
75 // . => ,
76 // : => +
77 // / => =
78
79 // 2. Process resulting string:
80 // split on first dot
81 // add "pairtree_root"
82 // then split everything else 2 chars at a time
83
84 // 3. Finally add in the (safely transformed) id:
85 // append directory that is prefix-removed id (transformed to be safe)
86 // further append 'id-safe'.json.bz
87
88 int id_dot_pos = id.indexOf(".");
89 String id_prefix = id.substring(0,id_dot_pos);
90 String id_tail = id.substring(id_dot_pos+1);
91 String id_tail_safe = id_tail.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
92
93 String [] pairs = id_tail_safe.split("(?<=\\G..)");
94 String joined_pairs = String.join("/", pairs);
95
96 String id_safe = id_prefix + "." + id_tail_safe;
97 String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
98 String filename = main_dir + "/" + id_tail_safe + "/" + id_safe + file_ext;
99
100 return filename;
101 }
102
103 protected void storeIDs(BufferedReader br)
104 {
105 long line_num = 1;
106 String line;
107
108 try {
109
110 System.err.print("Loading hashmap: ");
111 while ((line = br.readLine()) != null) {
112
113 String full_json_filename = line;
114 String json_filename_tail = full_filename_to_tail(full_json_filename);
115 String id = filename_tail_to_id(json_filename_tail);
116
117 id_check_.put(id, true);
118
119 if ((line_num % 100000) == 0) {
120 //System.err.println("sample id = " + id);
121 //System.err.println("Passed line: " + line_num);
122 System.err.print(".");
123 }
124 line_num++;
125
126 }
127 System.err.println(" => done.");
128 }
129 catch (Exception e) {
130 e.printStackTrace();
131 }
132
133 }
134 /**
135 * @see Servlet#init(ServletConfig)
136 */
137 public void init(ServletConfig config) throws ServletException {
138 super.init(config);
139
140 if (id_check_ == null) {
141 id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
142
143 String htrc_list_file = "htrc-ef-all-files.txt";
144 InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
145
146 try {
147 System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
148
149 InputStreamReader isr = new InputStreamReader(is, "UTF-8");
150 BufferedReader br = new BufferedReader(isr);
151
152 storeIDs(br);
153 br.close();
154 }
155 catch (Exception e) {
156 e.printStackTrace();
157 }
158 }
159 }
160
161 protected BufferedInputStream doRsyncDownload(String full_json_filename)
162 {
163 String json_filename_tail = full_filename_to_tail(full_json_filename);
164
165 BufferedInputStream bis = null;
166
167 Runtime runtime = Runtime.getRuntime();
168 String[] rsync_command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
169
170 try {
171 Process proc = runtime.exec(rsync_command);
172 proc.waitFor();
173 //System.err.println("*** Rsync finished");
174
175 FileInputStream fis = new FileInputStream(json_filename_tail);
176 bis = new BufferedInputStream(fis);
177
178 }
179 catch (Exception e) {
180 e.printStackTrace();
181 }
182
183 return bis;
184
185 }
186
187 protected void doCollectionToWorkset(HttpServletResponse response, String c, String a, String format) throws IOException
188 {
189 String post_url_params = "c="+c+"&a="+a+"&format="+format;
190
191 byte[] post_data = post_url_params.getBytes(StandardCharsets.UTF_8);
192 int post_data_len = post_data.length;
193
194 try {
195
196 URL post_url = new URL(ht_col_url);
197 HttpURLConnection conn = (HttpURLConnection) post_url.openConnection();
198 conn.setDoOutput(true);
199 conn.setInstanceFollowRedirects(false);
200 conn.setRequestMethod("POST");
201 conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
202 conn.setRequestProperty("charset", "utf-8");
203 conn.setRequestProperty("Content-Length", Integer.toString(post_data_len));
204 conn.setUseCaches(false);
205
206 try(DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) {
207 dos.write(post_data);
208 }
209 // try-resource auto-closes stream
210
211 InputStream is = conn.getInputStream();
212 InputStreamReader isr = new InputStreamReader(is);
213 BufferedReader reader = new BufferedReader(isr);
214
215 StringBuilder sb = new StringBuilder();
216 String line = null;
217 while ((line = reader.readLine()) != null)
218 {
219 sb.append(line + "\n");
220 }
221
222 response.setContentType("text/plain");
223 PrintWriter pw = response.getWriter();
224 pw.append(sb.toString());
225 }
226 catch (Exception e) {
227 e.printStackTrace();
228 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Failed to convert HT collection to HTRC workset");
229 }
230
231 }
232 /**
233 * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
234 */
235 protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
236 {
237
238
239 String cgi_ids = request.getParameter("ids");
240 String cgi_id = request.getParameter("id");
241 String cgi_download_id = request.getParameter("download-id");
242 String cgi_convert_col = request.getParameter("convert-col");
243
244 if (cgi_ids != null) {
245 response.setContentType("application/json");
246 PrintWriter pw = response.getWriter();
247
248 String[] ids = cgi_ids.split(",");
249 int ids_len = ids.length;
250
251 pw.append("{");
252
253 for (int i=0; i<ids_len; i++) {
254 String id = ids[i];
255
256 boolean exists = id_check_.get(id);
257
258 if (i>0) {
259 pw.append(",");
260 }
261 pw.append("\"" + id + "\":" + exists );
262 }
263 pw.append("}");
264
265 }
266 else if (cgi_id != null) {
267 response.setContentType("application/json");
268 PrintWriter pw = response.getWriter();
269
270 String id = cgi_id;
271 boolean exists = id_check_.get(id);
272 pw.append("{'" + id + "':" + exists + "}");
273 }
274 else if (cgi_download_id != null) {
275 String download_id = cgi_download_id;
276 boolean exists = id_check_.get(download_id);
277 if (!exists) {
278 // Error
279 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
280 }
281 else {
282 // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
283 String full_json_filename = id_to_pairtree_filename(download_id);
284
285 BufferedInputStream bis = doRsyncDownload(full_json_filename);
286
287 if (bis == null) {
288 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Rsync failed");
289 }
290 else {
291 String json_filename_tail = full_filename_to_tail(full_json_filename);
292
293 response.setContentType("application/x-bzip2");
294 response.setHeader("Content-Disposition", "attachment; filename=\"" + json_filename_tail + "\"");
295
296 OutputStream os=response.getOutputStream();
297 BufferedOutputStream bos = new BufferedOutputStream(os);
298
299 byte[] buf = new byte[1024];
300
301 while (true) {
302 int num_bytes = bis.read(buf);
303 if (num_bytes == -1) {
304 break;
305 }
306 bos.write(buf,0,num_bytes);
307 //total_num_bytes += num_bytes;
308 }
309
310 bis.close();
311 bos.close();
312 }
313 }
314 }
315 else if (cgi_convert_col != null) {
316 // c=464226859&a=download&format=text
317 String cgi_c = request.getParameter("c");
318 String cgi_a = request.getParameter("a");
319 String cgi_format = request.getParameter("format");
320
321 if ((cgi_c == null) || (cgi_a == null) || (cgi_format == null)) {
322 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed arguments. Need 'c', 'a', and 'format'");
323 }
324 else {
325 doCollectionToWorkset(response,cgi_c,cgi_a,cgi_format);
326 }
327
328 }
329 else {
330 PrintWriter pw = response.getWriter();
331
332 pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
333
334 }
335 //pw.close();
336
337 }
338
339 /**
340 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
341 */
342 protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
343 doGet(request, response);
344 }
345
346}
Note: See TracBrowser for help on using the repository browser.