source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31353

Last change on this file since 31353 was 31353, checked in by davidb, 7 years ago

Added debug print statement
  • Property svn:executable set to *
File size: 11.1 KB
Line 
1package org.hathitrust.extractedfeatures;
2
3import java.io.BufferedInputStream;
4import java.io.BufferedOutputStream;
5import java.io.BufferedReader;
6import java.io.DataOutputStream;
7import java.io.FileInputStream;
8import java.io.FileReader;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.InputStreamReader;
12import java.io.OutputStream;
13import java.io.PrintWriter;
14import java.io.UnsupportedEncodingException;
15import java.net.HttpURLConnection;
16import java.net.URL;
17import java.nio.charset.StandardCharsets;
18import java.nio.file.Files;
19import java.nio.file.Path;
20import java.nio.file.Paths;
21import java.util.ArrayList;
22import java.util.HashMap;
23
24import javax.servlet.ServletConfig;
25import javax.servlet.ServletException;
26import javax.servlet.annotation.WebServlet;
27import javax.servlet.http.HttpServlet;
28import javax.servlet.http.HttpServletRequest;
29import javax.servlet.http.HttpServletResponse;
30
31
32/**
33 * Servlet implementation class VolumeCheck
34 */
35@WebServlet("/VolumeCheck")
36public class VolumeCheck extends HttpServlet {
37 private static final long serialVersionUID = 1L;
38
39 protected static int HASHMAP_INIT_SIZE = 13800000;
40 protected static HashMap<String,Boolean> id_check_ = null;
41
42 protected static final String file_ext = ".json.bz2";
43 protected static final String ht_col_url = "https://babel.hathitrust.org/cgi/mb";
44
45 public VolumeCheck()
46 {}
47
48 protected static String full_filename_to_tail(String full_filename)
49 {
50 String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
51 return filename_tail;
52 }
53
54 protected static String filename_tail_to_id(String filename_tail)
55 {
56 String id = null;
57 if (filename_tail.endsWith(file_ext)) {
58 id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
59 }
60 else {
61 id = filename_tail;
62 }
63
64 id = id.replaceAll("\\+", ":").replaceAll("=", "/");
65
66 return id;
67 }
68
69 protected static String id_to_pairtree_filename(String id) {
70 // Example :-
71 // id: miun.adx6300.0001.001
72 // pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
73
74 // 1. Map 'difficult' chars:
75 // . => ,
76 // : => +
77 // / => =
78
79 // 2. Process resulting string:
80 // split on first dot
81 // add "pairtree_root"
82 // then split everything else 2 chars at a time
83
84 // 3. Finally add in the (safely transformed) id:
85 // append directory that is prefix-removed id (transformed to be safe)
86 // further append 'id-safe'.json.bz
87
88 int id_dot_pos = id.indexOf(".");
89 String id_prefix = id.substring(0,id_dot_pos);
90 String id_tail = id.substring(id_dot_pos+1);
91 String id_tail_safe = id_tail.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
92
93 String [] pairs = id_tail_safe.split("(?<=\\G..)");
94 String joined_pairs = String.join("/", pairs);
95
96 String id_safe = id_prefix + "." + id_tail_safe;
97 String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
98 String filename = main_dir + "/" + id_tail_safe + "/" + id_safe + file_ext;
99
100 return filename;
101 }
102
103 protected void storeIDs(BufferedReader br)
104 {
105 long line_num = 1;
106 String line;
107
108 try {
109
110 System.err.print("Loading hashmap: ");
111 while ((line = br.readLine()) != null) {
112
113 String full_json_filename = line;
114 String json_filename_tail = full_filename_to_tail(full_json_filename);
115 String id = filename_tail_to_id(json_filename_tail);
116
117 id_check_.put(id, true);
118
119 if ((line_num % 100000) == 0) {
120 //System.err.println("sample id = " + id);
121 //System.err.println("Passed line: " + line_num);
122 System.err.print(".");
123 }
124 line_num++;
125
126 }
127 System.err.println(" => done.");
128 }
129 catch (Exception e) {
130 e.printStackTrace();
131 }
132
133 }
134 /**
135 * @see Servlet#init(ServletConfig)
136 */
137 public void init(ServletConfig config) throws ServletException {
138 super.init(config);
139
140 if (id_check_ == null) {
141 id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
142
143 String htrc_list_file = "htrc-ef-all-files.txt";
144 InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
145
146 try {
147 System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
148
149 InputStreamReader isr = new InputStreamReader(is, "UTF-8");
150 BufferedReader br = new BufferedReader(isr);
151
152 storeIDs(br);
153 br.close();
154 }
155 catch (Exception e) {
156 e.printStackTrace();
157 }
158 }
159 }
160
161 protected BufferedInputStream doRsyncDownload(String full_json_filename)
162 {
163 String json_filename_tail = full_filename_to_tail(full_json_filename);
164
165 BufferedInputStream bis = null;
166
167 Runtime runtime = Runtime.getRuntime();
168 String[] rsync_command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
169
170 try {
171 Process proc = runtime.exec(rsync_command);
172 proc.waitFor();
173 //System.err.println("*** Rsync finished");
174
175 FileInputStream fis = new FileInputStream(json_filename_tail);
176 bis = new BufferedInputStream(fis);
177
178 }
179 catch (Exception e) {
180 e.printStackTrace();
181 }
182
183 return bis;
184
185 }
186
187 protected void doCollectionToWorkset(HttpServletResponse response, String c, String a, String format) throws IOException
188 {
189 String post_url_params = "c="+c+"&a="+a+"&format="+format;
190
191 byte[] post_data = post_url_params.getBytes(StandardCharsets.UTF_8);
192 int post_data_len = post_data.length;
193
194 try {
195
196 URL post_url = new URL(ht_col_url);
197 HttpURLConnection conn = (HttpURLConnection) post_url.openConnection();
198 conn.setDoOutput(true);
199 conn.setInstanceFollowRedirects(false);
200 conn.setRequestMethod("POST");
201 conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
202 conn.setRequestProperty("charset", "utf-8");
203 conn.setRequestProperty("Content-Length", Integer.toString(post_data_len));
204 conn.setUseCaches(false);
205
206 try(DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) {
207 dos.write(post_data);
208 }
209 // try-resource auto-closes stream
210
211 InputStream is = conn.getInputStream();
212 InputStreamReader isr = new InputStreamReader(is);
213 BufferedReader reader = new BufferedReader(isr);
214
215 StringBuilder workset_friendly_sb = new StringBuilder();
216 StringBuilder workset_unfriendly_sb = new StringBuilder();
217
218 String line = null;
219 int ci = 0;
220 while ((line = reader.readLine()) != null)
221 {
222 if (ci==0) {
223 workset_friendly_sb.append("#" + line + "\n");
224 }
225 else {
226 int first_tab_pos=line.indexOf("\t");
227 String id = (first_tab_pos>0) ? line.substring(0, first_tab_pos) : line;
228
229 if (id_check_.containsKey(id)) {
230 workset_friendly_sb.append(line + "\n");
231 }
232 else {
233 workset_unfriendly_sb.append("#" + line + "\n");
234 }
235 }
236
237 ci++;
238 }
239
240 response.setContentType("text/plain");
241 PrintWriter pw = response.getWriter();
242 pw.append(workset_friendly_sb.toString());
243 pw.append("## The following volumes are not in the HTRC Extracted Feature dataset\n");
244 pw.append(workset_unfriendly_sb.toString());
245 }
246 catch (Exception e) {
247 e.printStackTrace();
248 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Failed to convert HT collection to HTRC workset");
249 }
250
251 }
252 /**
253 * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
254 */
255 protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
256 {
257
258
259 String cgi_ids = request.getParameter("ids");
260 String cgi_id = request.getParameter("id");
261 String cgi_download_id = request.getParameter("download-id");
262 String cgi_convert_col = request.getParameter("convert-col");
263
264 System.err.println("**** cgi_ids = '" + cgi_ids + "'");
265 System.err.println("**** cgi_convert_col = '" + cgi_convert_col + "'");
266
267 if (cgi_ids != null) {
268 response.setContentType("application/json");
269 PrintWriter pw = response.getWriter();
270
271 String[] ids = cgi_ids.split(",");
272 int ids_len = ids.length;
273
274 pw.append("{");
275
276 for (int i=0; i<ids_len; i++) {
277 String id = ids[i];
278
279 boolean exists = id_check_.get(id);
280
281 if (i>0) {
282 pw.append(",");
283 }
284 pw.append("\"" + id + "\":" + exists );
285 }
286 pw.append("}");
287
288 }
289 else if (cgi_id != null) {
290 response.setContentType("application/json");
291 PrintWriter pw = response.getWriter();
292
293 String id = cgi_id;
294 boolean exists = id_check_.get(id);
295 pw.append("{'" + id + "':" + exists + "}");
296 }
297 else if (cgi_download_id != null) {
298 String download_id = cgi_download_id;
299 boolean exists = id_check_.get(download_id);
300 if (!exists) {
301 // Error
302 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
303 }
304 else {
305 // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
306 String full_json_filename = id_to_pairtree_filename(download_id);
307
308 BufferedInputStream bis = doRsyncDownload(full_json_filename);
309
310 if (bis == null) {
311 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Rsync failed");
312 }
313 else {
314 String json_filename_tail = full_filename_to_tail(full_json_filename);
315
316 response.setContentType("application/x-bzip2");
317 response.setHeader("Content-Disposition", "attachment; filename=\"" + json_filename_tail + "\"");
318
319 OutputStream os=response.getOutputStream();
320 BufferedOutputStream bos = new BufferedOutputStream(os);
321
322 byte[] buf = new byte[1024];
323
324 while (true) {
325 int num_bytes = bis.read(buf);
326 if (num_bytes == -1) {
327 break;
328 }
329 bos.write(buf,0,num_bytes);
330 //total_num_bytes += num_bytes;
331 }
332
333 bis.close();
334 bos.close();
335 }
336 }
337 }
338 else if (cgi_convert_col != null) {
339 // c=464226859&a=download&format=text
340 //String cgi_c = request.getParameter("c");
341 String cgi_a = request.getParameter("a");
342 String cgi_format = request.getParameter("format");
343
344 if ((cgi_a == null) || (cgi_format == null)) {
345 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed arguments. Need 'a', and 'format'");
346 }
347 else {
348 doCollectionToWorkset(response,cgi_convert_col,cgi_a,cgi_format);
349 }
350
351 }
352 else {
353 PrintWriter pw = response.getWriter();
354
355 pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
356
357 }
358 //pw.close();
359
360 }
361
362 /**
363 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
364 */
365 protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
366 doGet(request, response);
367 }
368
369}
Note: See TracBrowser for help on using the repository browser.