source: other-projects/hathitrust/wcsa/vol-checker/src/org/hathitrust/extractedfeatures/VolumeCheck.java@ 31348

Last change on this file since 31348 was 31348, checked in by davidb, 7 years ago

Restructure of how convert-to works
  • Property svn:executable set to *
File size: 10.4 KB
Line 
1package org.hathitrust.extractedfeatures;
2
3import java.io.BufferedInputStream;
4import java.io.BufferedOutputStream;
5import java.io.BufferedReader;
6import java.io.DataOutputStream;
7import java.io.FileInputStream;
8import java.io.FileReader;
9import java.io.IOException;
10import java.io.InputStream;
11import java.io.InputStreamReader;
12import java.io.OutputStream;
13import java.io.PrintWriter;
14import java.io.UnsupportedEncodingException;
15import java.net.HttpURLConnection;
16import java.net.URL;
17import java.nio.charset.StandardCharsets;
18import java.nio.file.Files;
19import java.nio.file.Path;
20import java.nio.file.Paths;
21import java.util.ArrayList;
22import java.util.HashMap;
23
24import javax.servlet.ServletConfig;
25import javax.servlet.ServletException;
26import javax.servlet.annotation.WebServlet;
27import javax.servlet.http.HttpServlet;
28import javax.servlet.http.HttpServletRequest;
29import javax.servlet.http.HttpServletResponse;
30
31
32/**
33 * Servlet implementation class VolumeCheck
34 */
35@WebServlet("/VolumeCheck")
36public class VolumeCheck extends HttpServlet {
37 private static final long serialVersionUID = 1L;
38
39 protected static int HASHMAP_INIT_SIZE = 13800000;
40 protected static HashMap<String,Boolean> id_check_ = null;
41
42 protected static final String file_ext = ".json.bz2";
43 protected static final String ht_col_url = "https://babel.hathitrust.org/cgi/mb";
44
45 public VolumeCheck()
46 {}
47
48 protected static String full_filename_to_tail(String full_filename)
49 {
50 String filename_tail = full_filename.substring(full_filename.lastIndexOf("/")+1);
51 return filename_tail;
52 }
53
54 protected static String filename_tail_to_id(String filename_tail)
55 {
56 String id = null;
57 if (filename_tail.endsWith(file_ext)) {
58 id = filename_tail.substring(0,filename_tail.lastIndexOf(file_ext));
59 }
60 else {
61 id = filename_tail;
62 }
63
64 id = id.replaceAll("\\+", ":").replaceAll("=", "/");
65
66 return id;
67 }
68
69 protected static String id_to_pairtree_filename(String id) {
70 // Example :-
71 // id: miun.adx6300.0001.001
72 // pairtree filename: miun/pairtree_root/ad/x6/30/0,/00/01/,0/01/adx6300,0001,001/miun.adx6300,0001,001.json.bz2
73
74 // 1. Map 'difficult' chars:
75 // . => ,
76 // : => +
77 // / => =
78
79 // 2. Process resulting string:
80 // split on first dot
81 // add "pairtree_root"
82 // then split everything else 2 chars at a time
83
84 // 3. Finally add in the (safely transformed) id:
85 // append directory that is prefix-removed id (transformed to be safe)
86 // further append 'id-safe'.json.bz
87
88 int id_dot_pos = id.indexOf(".");
89 String id_prefix = id.substring(0,id_dot_pos);
90 String id_tail = id.substring(id_dot_pos+1);
91 String id_tail_safe = id_tail.replaceAll("\\.", ",").replaceAll(":", "+").replaceAll("/", "=");
92
93 String [] pairs = id_tail_safe.split("(?<=\\G..)");
94 String joined_pairs = String.join("/", pairs);
95
96 String id_safe = id_prefix + "." + id_tail_safe;
97 String main_dir = id_prefix + "/pairtree_root/" + joined_pairs;
98 String filename = main_dir + "/" + id_tail_safe + "/" + id_safe + file_ext;
99
100 return filename;
101 }
102
103 protected void storeIDs(BufferedReader br)
104 {
105 long line_num = 1;
106 String line;
107
108 try {
109
110 System.err.print("Loading hashmap: ");
111 while ((line = br.readLine()) != null) {
112
113 String full_json_filename = line;
114 String json_filename_tail = full_filename_to_tail(full_json_filename);
115 String id = filename_tail_to_id(json_filename_tail);
116
117 id_check_.put(id, true);
118
119 if ((line_num % 100000) == 0) {
120 //System.err.println("sample id = " + id);
121 //System.err.println("Passed line: " + line_num);
122 System.err.print(".");
123 }
124 line_num++;
125
126 }
127 System.err.println(" => done.");
128 }
129 catch (Exception e) {
130 e.printStackTrace();
131 }
132
133 }
134 /**
135 * @see Servlet#init(ServletConfig)
136 */
137 public void init(ServletConfig config) throws ServletException {
138 super.init(config);
139
140 if (id_check_ == null) {
141 id_check_ = new HashMap<String,Boolean>(HASHMAP_INIT_SIZE);
142
143 String htrc_list_file = "htrc-ef-all-files.txt";
144 InputStream is = getServletContext().getResourceAsStream("/WEB-INF/" + htrc_list_file);
145
146 try {
147 System.err.println("INFO: Loading in volume IDS: " + htrc_list_file);
148
149 InputStreamReader isr = new InputStreamReader(is, "UTF-8");
150 BufferedReader br = new BufferedReader(isr);
151
152 storeIDs(br);
153 br.close();
154 }
155 catch (Exception e) {
156 e.printStackTrace();
157 }
158 }
159 }
160
161 protected BufferedInputStream doRsyncDownload(String full_json_filename)
162 {
163 String json_filename_tail = full_filename_to_tail(full_json_filename);
164
165 BufferedInputStream bis = null;
166
167 Runtime runtime = Runtime.getRuntime();
168 String[] rsync_command = {"rsync","-av","data.analytics.hathitrust.org::features/" + full_json_filename, "."};
169
170 try {
171 Process proc = runtime.exec(rsync_command);
172 proc.waitFor();
173 //System.err.println("*** Rsync finished");
174
175 FileInputStream fis = new FileInputStream(json_filename_tail);
176 bis = new BufferedInputStream(fis);
177
178 }
179 catch (Exception e) {
180 e.printStackTrace();
181 }
182
183 return bis;
184
185 }
186
187 protected void doCollectionToWorkset(HttpServletResponse response, String c, String a, String format) throws IOException
188 {
189 String post_url_params = "c="+c+"&a="+a+"&format="+format;
190
191 byte[] post_data = post_url_params.getBytes(StandardCharsets.UTF_8);
192 int post_data_len = post_data.length;
193
194 try {
195
196 URL post_url = new URL(ht_col_url);
197 HttpURLConnection conn = (HttpURLConnection) post_url.openConnection();
198 conn.setDoOutput(true);
199 conn.setInstanceFollowRedirects(false);
200 conn.setRequestMethod("POST");
201 conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
202 conn.setRequestProperty("charset", "utf-8");
203 conn.setRequestProperty("Content-Length", Integer.toString(post_data_len));
204 conn.setUseCaches(false);
205
206 try(DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) {
207 dos.write(post_data);
208 }
209 // try-resource auto-closes stream
210
211 InputStream is = conn.getInputStream();
212 InputStreamReader isr = new InputStreamReader(is);
213 BufferedReader reader = new BufferedReader(isr);
214
215 StringBuilder sb = new StringBuilder();
216 String line = null;
217 while ((line = reader.readLine()) != null)
218 {
219 sb.append(line + "\n");
220 }
221
222 response.setContentType("text/plain");
223 PrintWriter pw = response.getWriter();
224 pw.append(sb.toString());
225 }
226 catch (Exception e) {
227 e.printStackTrace();
228 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Failed to convert HT collection to HTRC workset");
229 }
230
231 }
232 /**
233 * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
234 */
235 protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
236 {
237
238
239 String cgi_ids = request.getParameter("ids");
240 String cgi_id = request.getParameter("id");
241 String cgi_download_id = request.getParameter("download-id");
242 String cgi_convert_col = request.getParameter("convert-col");
243
244 System.err.println("**** cgi_convert_col = '" + cgi_convert_col + "'");
245
246 if (cgi_ids != null) {
247 response.setContentType("application/json");
248 PrintWriter pw = response.getWriter();
249
250 String[] ids = cgi_ids.split(",");
251 int ids_len = ids.length;
252
253 pw.append("{");
254
255 for (int i=0; i<ids_len; i++) {
256 String id = ids[i];
257
258 boolean exists = id_check_.get(id);
259
260 if (i>0) {
261 pw.append(",");
262 }
263 pw.append("\"" + id + "\":" + exists );
264 }
265 pw.append("}");
266
267 }
268 else if (cgi_id != null) {
269 response.setContentType("application/json");
270 PrintWriter pw = response.getWriter();
271
272 String id = cgi_id;
273 boolean exists = id_check_.get(id);
274 pw.append("{'" + id + "':" + exists + "}");
275 }
276 else if (cgi_download_id != null) {
277 String download_id = cgi_download_id;
278 boolean exists = id_check_.get(download_id);
279 if (!exists) {
280 // Error
281 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "The requested volume id does not exist.");
282 }
283 else {
284 // rsync -av data.analytics.hathitrust.org::features/{PATH-TO-FILE} .
285 String full_json_filename = id_to_pairtree_filename(download_id);
286
287 BufferedInputStream bis = doRsyncDownload(full_json_filename);
288
289 if (bis == null) {
290 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Rsync failed");
291 }
292 else {
293 String json_filename_tail = full_filename_to_tail(full_json_filename);
294
295 response.setContentType("application/x-bzip2");
296 response.setHeader("Content-Disposition", "attachment; filename=\"" + json_filename_tail + "\"");
297
298 OutputStream os=response.getOutputStream();
299 BufferedOutputStream bos = new BufferedOutputStream(os);
300
301 byte[] buf = new byte[1024];
302
303 while (true) {
304 int num_bytes = bis.read(buf);
305 if (num_bytes == -1) {
306 break;
307 }
308 bos.write(buf,0,num_bytes);
309 //total_num_bytes += num_bytes;
310 }
311
312 bis.close();
313 bos.close();
314 }
315 }
316 }
317 else if (cgi_convert_col != null) {
318 // c=464226859&a=download&format=text
319 //String cgi_c = request.getParameter("c");
320 String cgi_a = request.getParameter("a");
321 String cgi_format = request.getParameter("format");
322
323 if ((cgi_a == null) || (cgi_format == null)) {
324 response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed arguments. Need 'a', and 'format'");
325 }
326 else {
327 doCollectionToWorkset(response,cgi_convert_col,cgi_a,cgi_format);
328 }
329
330 }
331 else {
332 PrintWriter pw = response.getWriter();
333
334 pw.append("General Info: Number of HTRC Volumes in check-list = " + id_check_.size());
335
336 }
337 //pw.close();
338
339 }
340
341 /**
342 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
343 */
344 protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
345 doGet(request, response);
346 }
347
348}
Note: See TracBrowser for help on using the repository browser.