source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31352

Last change on this file since 31352 was 31352, checked in by davidb, 7 years ago

collection-to-workset now with id-check added to filter
  • Property svn:executable set to *
File size: 11.1 KB
Line 
1package org.hathitrust.extractedfeatures;
2
3import java.io.BufferedInputStream;
4import java.io.BufferedOutputStream;
5import java.io.BufferedReader;
6import java.io.DataOutputStream;
7import java.io.FileInputStream;
8import java.io.FileReader;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.InputStreamReader;
12import java.io.OutputStream;
13import java.io.PrintWriter;
14import java.io.UnsupportedEncodingException;
15import java.net.HttpURLConnection;
16import java.net.URL;
17import java.nio.charset.StandardCharsets;
18import java.nio.file.Files;
19import java.nio.file.Path;
20import java.nio.file.Paths;
21import java.util.ArrayList;
22import java.util.HashMap;
23
24import javax.servlet.ServletConfig;
25import javax.servlet.ServletException;
26import javax.servlet.annotation.WebServlet;
27import javax.servlet.http.HttpServlet;
28import javax.servlet.http.HttpServletRequest;
29import javax.servlet.http.HttpServletResponse;
30
31
32/**
33 * Servlet implementation class VolumeCheck
34 */
35@WebServlet("/VolumeCheck")
36public class VolumeCheck extends HttpServlet {
37 private static final long serialVersionUID = 1L;
38
39 protected static int HASHMAP_INIT_SIZE = 13800000;
40 protected static HashMap<String,Boolean> id_check_ = null;
41
42 protected static final String file_ext = ".json.bz2";
43 protected static final String ht_col_url = "https://babel.hathitrust.org/cgi/mb";
44
45 public VolumeCheck()
46 {}
47
48 protected static String full_filename_to_tail(String full_filename)
49 {
50 String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
51 return filename_tail;
52 }
53
54 protected static String filename_tail_to_id(String filename_tail)
55 {
56 String id = null;
57 if (filename_tail.endsWith(file_ext)) {
58 id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
59 }
60 else {
61 id = filename_tail;
62 }
63
64 id = id.replaceAll("\\+", ":").replaceAll("=", "/");
65
66 return id;
67 }
68
69 protected static String id_to_pairtree_filename(String id) {
70 // Example :-
71 // id: miun.adx6300.0001.001
72 // pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
73
74 // 1. Map 'difficult' chars:
75 // . => ,
76 // : => +
77 // / => =
78
79 // 2. Process resulting string:
80 // split on first dot
81 // add "pairtree_root"
82 // then split everything else 2 chars at a time
83
84 // 3. Finally add in the (safely transformed) id:
85 // append directory that is prefix-removed id (transformed to be safe)
86 // further append 'id-safe'.json.bz
87
88 int id_dot_pos = id.indexOf(".");
89 String id_prefix = id.substring(0,id_dot_pos);
90 String id_tail = id.substring(id_dot_pos+1);
91 String id_tail_safe = id_tail.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
92
93 String [] pairs = id_tail_safe.split("(?<=\\G..)");
94 String joined_pairs = String.join("/", pairs);
95
96 String id_safe = id_prefix + "." + id_tail_safe;
97 String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
98 String filename = main_dir + "/" + id_tail_safe + "/" + id_safe + file_ext;
99
100 return filename;
101 }
102
103 protected void storeIDs(BufferedReader br)
104 {
105 long line_num = 1;
106 String line;
107
108 try {
109
110 System.err.print("Loading hashmap: ");
111 while ((line = br.readLine()) != null) {
112
113 String full_json_filename = line;
114 String json_filename_tail = full_filename_to_tail(full_json_filename);
115 String id = filename_tail_to_id(json_filename_tail);
116
117 id_check_.put(id, true);
118
119 if ((line_num % 100000) == 0) {
120 //System.err.println("sample id = " + id);
121 //System.err.println("Passed line: " + line_num);
122 System.err.print(".");
123 }
124 line_num++;
125
126 }
127 System.err.println(" => done.");
128 }
129 catch (Exception e) {
130 e.printStackTrace();
131 }
132
133 }
134 /**
135 * @see Servlet#init(ServletConfig)
136 */
137 public void init(ServletConfig config) throws ServletException {
138 super.init(config);
139
140 if (id_check_ == null) {
141 id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
142
143 String htrc_list_file = "htrc-ef-all-files.txt";
144 InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
145
146 try {
147 System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
148
149 InputStreamReader isr = new InputStreamReader(is, "UTF-8");
150 BufferedReader br = new BufferedReader(isr);
151
152 storeIDs(br);
153 br.close();
154 }
155 catch (Exception e) {
156 e.printStackTrace();
157 }
158 }
159 }
160
161 protected BufferedInputStream doRsyncDownload(String full_json_filename)
162 {
163 String json_filename_tail = full_filename_to_tail(full_json_filename);
164
165 BufferedInputStream bis = null;
166
167 Runtime runtime = Runtime.getRuntime();
168 String[] rsync_command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
169
170 try {
171 Process proc = runtime.exec(rsync_command);
172 proc.waitFor();
173 //System.err.println("*** Rsync finished");
174
175 FileInputStream fis = new FileInputStream(json_filename_tail);
176 bis = new BufferedInputStream(fis);
177
178 }
179 catch (Exception e) {
180 e.printStackTrace();
181 }
182
183 return bis;
184
185 }
186
187 protected void doCollectionToWorkset(HttpServletResponse response, String c, String a, String format) throws IOException
188 {
189 String post_url_params = "c="+c+"&a="+a+"&format="+format;
190
191 byte[] post_data = post_url_params.getBytes(StandardCharsets.UTF_8);
192 int post_data_len = post_data.length;
193
194 try {
195
196 URL post_url = new URL(ht_col_url);
197 HttpURLConnection conn = (HttpURLConnection) post_url.openConnection();
198 conn.setDoOutput(true);
199 conn.setInstanceFollowRedirects(false);
200 conn.setRequestMethod("POST");
201 conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
202 conn.setRequestProperty("charset", "utf-8");
203 conn.setRequestProperty("Content-Length", Integer.toString(post_data_len));
204 conn.setUseCaches(false);
205
206 try(DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) {
207 dos.write(post_data);
208 }
209 // try-resource auto-closes stream
210
211 InputStream is = conn.getInputStream();
212 InputStreamReader isr = new InputStreamReader(is);
213 BufferedReader reader = new BufferedReader(isr);
214
215 StringBuilder workset_friendly_sb = new StringBuilder();
216 StringBuilder workset_unfriendly_sb = new StringBuilder();
217
218 String line = null;
219 int ci = 0;
220 while ((line = reader.readLine()) != null)
221 {
222 if (ci==0) {
223 workset_friendly_sb.append("#" + line + "\n");
224 }
225 else {
226 int first_tab_pos=line.indexOf("\t");
227 String id = (first_tab_pos>0) ? line.substring(0, first_tab_pos) : line;
228
229 if (id_check_.containsKey(id)) {
230 workset_friendly_sb.append(line + "\n");
231 }
232 else {
233 workset_unfriendly_sb.append("#" + line + "\n");
234 }
235 }
236
237 ci++;
238 }
239
240 response.setContentType("text/plain");
241 PrintWriter pw = response.getWriter();
242 pw.append(workset_friendly_sb.toString());
243 pw.append("## The following volumes are not in the HTRC Extracted Feature dataset\n");
244 pw.append(workset_unfriendly_sb.toString());
245 }
246 catch (Exception e) {
247 e.printStackTrace();
248 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Failed to convert HT collection to HTRC workset");
249 }
250
251 }
252 /**
253 * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
254 */
255 protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
256 {
257
258
259 String cgi_ids = request.getParameter("ids");
260 String cgi_id = request.getParameter("id");
261 String cgi_download_id = request.getParameter("download-id");
262 String cgi_convert_col = request.getParameter("convert-col");
263
264 //System.err.println("**** cgi_convert_col = '" + cgi_convert_col + "'");
265
266 if (cgi_ids != null) {
267 response.setContentType("application/json");
268 PrintWriter pw = response.getWriter();
269
270 String[] ids = cgi_ids.split(",");
271 int ids_len = ids.length;
272
273 pw.append("{");
274
275 for (int i=0; i<ids_len; i++) {
276 String id = ids[i];
277
278 boolean exists = id_check_.get(id);
279
280 if (i>0) {
281 pw.append(",");
282 }
283 pw.append("\"" + id + "\":" + exists );
284 }
285 pw.append("}");
286
287 }
288 else if (cgi_id != null) {
289 response.setContentType("application/json");
290 PrintWriter pw = response.getWriter();
291
292 String id = cgi_id;
293 boolean exists = id_check_.get(id);
294 pw.append("{'" + id + "':" + exists + "}");
295 }
296 else if (cgi_download_id != null) {
297 String download_id = cgi_download_id;
298 boolean exists = id_check_.get(download_id);
299 if (!exists) {
300 // Error
301 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
302 }
303 else {
304 // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
305 String full_json_filename = id_to_pairtree_filename(download_id);
306
307 BufferedInputStream bis = doRsyncDownload(full_json_filename);
308
309 if (bis == null) {
310 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Rsync failed");
311 }
312 else {
313 String json_filename_tail = full_filename_to_tail(full_json_filename);
314
315 response.setContentType("application/x-bzip2");
316 response.setHeader("Content-Disposition", "attachment; filename=\"" + json_filename_tail + "\"");
317
318 OutputStream os=response.getOutputStream();
319 BufferedOutputStream bos = new BufferedOutputStream(os);
320
321 byte[] buf = new byte[1024];
322
323 while (true) {
324 int num_bytes = bis.read(buf);
325 if (num_bytes == -1) {
326 break;
327 }
328 bos.write(buf,0,num_bytes);
329 //total_num_bytes += num_bytes;
330 }
331
332 bis.close();
333 bos.close();
334 }
335 }
336 }
337 else if (cgi_convert_col != null) {
338 // c=464226859&a=download&format=text
339 //String cgi_c = request.getParameter("c");
340 String cgi_a = request.getParameter("a");
341 String cgi_format = request.getParameter("format");
342
343 if ((cgi_a == null) || (cgi_format == null)) {
344 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed arguments. Need 'a', and 'format'");
345 }
346 else {
347 doCollectionToWorkset(response,cgi_convert_col,cgi_a,cgi_format);
348 }
349
350 }
351 else {
352 PrintWriter pw = response.getWriter();
353
354 pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
355
356 }
357 //pw.close();
358
359 }
360
361 /**
362 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
363 */
364 protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
365 doGet(request, response);
366 }
367
368}
Note: See TracBrowser for help on using the repository browser.