source: main/trunk/greenstone3/src/java/org/greenstone/gsdl3/build/GS2PerlConstructor.java@ 30617

Last change on this file since 30617 was 30617, checked in by ak19, 8 years ago

Bugfix to oversight: web document editor didn't reindex on removing metadata, only on setting it. This is because there wasn't corresponding Service on Java side to handle removing metadata. Now the Java code has a ModifyMetdata service and this is connected to both the set- and remove-metadata functions in the javascript. For now, the javascript metadataserver calls have been made synchronous by setting the async ajax-jquery property to false. Need to test whether the code still works asynchronous as before.
  • Property svn:keywords set to Author Date Id Revision
File size: 16.6 KB
Line 
1package org.greenstone.gsdl3.build;
2
3// greenstome classes
4import org.greenstone.gsdl3.util.*;
5import org.greenstone.util.Misc;
6import org.greenstone.util.GlobalProperties;
7
8// xml classes
9import org.w3c.dom.Element;
10import org.w3c.dom.NodeList;
11
12//general java classes
13import java.io.BufferedReader;
14import java.io.BufferedWriter;
15import java.io.Closeable;
16import java.io.FileWriter;
17import java.io.InputStreamReader;
18import java.io.File;
19import java.io.IOException;
20import java.util.ArrayList;
21import java.util.Vector;
22
23import org.apache.log4j.*;
24
25/**
26 * CollectionConstructor class for greenstone 2 compatible building it uses the
27 * perl scripts to do the building stuff
28 */
29public class GS2PerlConstructor extends CollectionConstructor
30{
31 static Logger logger = Logger.getLogger(org.greenstone.gsdl3.build.GS2PerlConstructor.class.getName());
32
33 public static final int NEW = 0;
34 public static final int IMPORT = 1;
35 public static final int BUILD = 2;
36 public static final int ACTIVATE = 3;
37 public static final int MODIFY_METADATA_SERVER = 4;
38
39 /**
40 * gsdlhome for greenstone 2 - we use the perl modules and building scripts
41 * from there
42 */
43 protected String gsdl2home = null;
44 /** gsdlhome for gsdl3 - shouldn't need this eventually ?? */
45 protected String gsdl3home = null;
46 /** gsdlos for greenstone 2 */
47 protected String gsdlos = null;
48 /** the path environment variable */
49 protected String path = null;
50
51 public GS2PerlConstructor(String name)
52 {
53 super(name);
54 }
55
56 /** retrieves the necessary environment variables */
57 public boolean configure()
58 {
59 // try to get the environment variables
60 this.gsdl3home = GlobalProperties.getGSDL3Home();
61 this.gsdl2home = this.gsdl3home + File.separator + ".." + File.separator + "gs2build";
62 this.gsdlos = Misc.getGsdlOS();
63
64 this.path = System.getenv("PATH");
65
66 if (this.gsdl2home == null)
67 {
68 System.err.println("You must have gs2build installed, and GSDLHOME set for GS2Perl building to work!!");
69 return false;
70 }
71 if (this.gsdl3home == null || this.gsdlos == null)
72 {
73 System.err.println("You must have GSDL3HOME and GSDLOS set for GS2Perl building to work!!");
74 return false;
75 }
76 if (this.path == null)
77 {
78 System.err.println("You must have the PATH set for GS2Perl building to work!!");
79 return false;
80 }
81 return true;
82 }
83
84 public void run()
85 {
86 String msg;
87 ConstructionEvent evt;
88 if (this.process_type == -1)
89 {
90 msg = "Error: you must set the action type";
91 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
92 sendMessage(evt);
93 return;
94 }
95 if (this.site_home == null)
96 {
97 msg = "Error: you must set site_home";
98 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
99 sendMessage(evt);
100 return;
101 }
102 if (this.process_type != NEW && this.collection_name == null)
103 {
104 msg = "Error: you must set collection_name";
105 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
106 sendMessage(evt);
107 return;
108 }
109
110 switch (this.process_type)
111 {
112 case NEW:
113 newCollection();
114 break;
115 case IMPORT:
116 importCollection();
117 break;
118 case BUILD:
119 buildCollection();
120 break;
121 case ACTIVATE:
122 activateCollection();
123 break;
124 case MODIFY_METADATA_SERVER:
125 modifyMetadataForCollection();
126 break;
127 default:
128 msg = "wrong type of action specified!";
129 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
130 sendMessage(evt);
131 break;
132 }
133 }
134
135 protected void newCollection()
136 {
137 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: new collection."));
138 Vector<String> command = new Vector<String>();
139 command.add("gs2_mkcol.pl");
140 command.add("-site");
141 command.add(this.site_home);
142 command.add("-collectdir");
143 command.add(GSFile.collectDir(this.site_home));
144 command.addAll(extractParameters(this.process_params));
145 command.add(this.collection_name);
146 String[] command_str = {};
147 command_str = command.toArray(command_str);
148 if (runPerlCommand(command_str))
149 {
150 // success!! - need to send the final completed message
151 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
152 } // else an error message has already been sent, do nothing
153
154 }
155
156 protected void importCollection()
157 {
158 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: import collection."));
159 Vector<String> command = new Vector<String>();
160
161 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
162 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
163 {
164 perlPath = perlPath + File.separator;
165 }
166
167 command.add(perlPath + "perl");
168 command.add("-S");
169 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "import.pl");
170 if (this.manifest_file != null)
171 {
172 command.add("-keepold");
173 command.add("-manifest");
174 command.add(this.manifest_file);
175 }
176 command.add("-site");
177 command.add(this.site_name);
178 command.add("-collectdir");
179 command.add(GSFile.collectDir(this.site_home));
180 command.addAll(extractParameters(this.process_params));
181 command.add(this.collection_name);
182 String[] command_str = {};
183 command_str = command.toArray(command_str);
184
185 if (runPerlCommand(command_str))
186 {
187 // success!! - need to send the final completed message
188 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
189 } // else an error message has already been sent, do nothing
190 }
191
192 protected void buildCollection()
193 {
194 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: build collection."));
195 Vector<String> command = new Vector<String>();
196
197 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
198 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
199 {
200 perlPath = perlPath + File.separator;
201 }
202
203 command.add(perlPath + "perl");
204 command.add("-S");
205 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "incremental-buildcol.pl");
206 command.add("-incremental");
207 command.add("-builddir");
208 command.add(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator +"index");
209 command.add("-site");
210 command.add(this.site_name);
211 command.add("-collectdir");
212 command.add(GSFile.collectDir(this.site_home));
213// command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly
214 command.addAll(extractParameters(this.process_params));
215 command.add(this.collection_name);
216
217 String[] command_str = {};
218 command_str = command.toArray(command_str);
219
220 if (runPerlCommand(command_str))
221 {
222 // success!! - need to send the final completed message
223 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
224 }// else an error message has already been sent, do nothing
225 }
226
227 protected void activateCollection()
228 {
229 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: activate collection."));
230
231 // first check that we have a building directory
232 // (don't want to bother running activate.pl otherwise)
233 File build_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
234 if (!build_dir.exists())
235 {
236 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "build dir doesn't exist!"));
237 return;
238 }
239
240 /*
241
242 // move building to index
243 File index_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
244 if (index_dir.exists())
245 {
246 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "deleting index directory"));
247 GSFile.deleteFile(index_dir);
248 if (index_dir.exists())
249 {
250 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index directory still exists!"));
251 return;
252 }
253 }
254
255 GSFile.moveDirectory(build_dir, index_dir);
256 if (!index_dir.exists())
257 {
258 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index dir wasn't created!"));
259 }
260
261 // success!! - need to send the final completed message
262 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
263 */
264
265 // Running activate.pl instead of making java move building to index as above
266 // circumvents the issue of the jdbm .lg log file (managed by TransactionManager)
267 // in index dir not getting deleted at times. The perl code is able to delete this
268 // sucessfully consistently during testing, whereas java at times is unable to delete it.
269 Vector<String> command = new Vector<String>();
270
271 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
272 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
273 {
274 perlPath = perlPath + File.separator;
275 }
276
277 command.add(perlPath + "perl");
278 command.add("-S");
279 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "activate.pl");
280 command.add("-incremental");
281 command.add("-builddir");
282 command.add(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator +"index");
283 command.add("-site");
284 command.add(this.site_name);
285 command.add("-collectdir");
286 command.add(GSFile.collectDir(this.site_home));
287// command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly. Shouldn't be added for incremental building
288 command.add("-skipactivation"); // gsdl3/util/GS2Construct does the activation and reactivation
289 command.addAll(extractParameters(this.process_params));
290 command.add(this.collection_name);
291
292 String[] command_str = {};
293 command_str = command.toArray(command_str);
294
295 if (runPerlCommand(command_str))
296 {
297 // success!! - need to send the final completed message
298 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
299 }// else an error message has already been sent, do nothing
300
301 }
302
303
304 protected void modifyMetadataForCollection()
305 {
306 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection metadata: modifyMetadata (set or remove meta) for collection."));
307
308 Vector<String> command = new Vector<String>();
309
310 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
311 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
312 {
313 perlPath = perlPath + File.separator;
314 }
315
316 String cgi_directory = GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi";
317 command.add(perlPath + "perl");
318 command.add("-S");
319 //command.add(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi" + File.separator + "metadata-server.pl");
320 command.add(cgi_directory + File.separator + "metadata-server.pl");
321
322 // Need to set QUERY_STRING and REQUEST_METHOD=GET in environment
323 // http://www.cgi101.com/class/ch3/text.html
324 String[] envvars = {
325 "QUERY_STRING=" + this.query_string,
326 "REQUEST_METHOD=GET"
327 };
328
329 String[] command_str = {};
330 command_str = command.toArray(command_str);
331
332 // http://www.cgi101.com/class/ch3/text.html
333 // setenv QUERY_STRING and REQUEST_METHOD = GET.
334 if (runPerlCommand(command_str, envvars, new File(cgi_directory)))
335 //new File(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi")))
336 {
337 // success!! - need to send the final completed message
338 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
339 }// else an error message has already been sent, do nothing
340
341 }
342
343 /** extracts all the args from the xml and returns them in a Vector */
344 protected Vector<String> extractParameters(Element param_list)
345 {
346
347 Vector<String> args = new Vector<String>();
348 if (param_list == null)
349 {
350 return args; // return an empty vector
351 }
352 NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
353
354 for (int i = 0; i < params.getLength(); i++)
355 {
356 Element p = (Element) params.item(i);
357 String name = p.getAttribute(GSXML.NAME_ATT);
358 String value = p.getAttribute(GSXML.VALUE_ATT);
359 if (!name.equals(""))
360 {
361 args.add("-" + name);
362 if (!value.equals(""))
363 {
364 args.add(value);
365 }
366 }
367 }
368
369 return args;
370 }
371
372 /** returns true if completed correctly, false otherwise */
373 protected boolean runPerlCommand(String[] command) {
374 return runPerlCommand(command, null, null);
375 }
376
377 protected boolean runPerlCommand(String[] command, String[] envvars, File dir)
378 {
379 boolean success = true;
380
381 int sepIndex = this.gsdl3home.lastIndexOf(File.separator);
382 String srcHome = this.gsdl3home.substring(0, sepIndex);
383
384 ArrayList<String> args = new ArrayList<String>();
385 args.add("GSDLHOME=" + this.gsdl2home);
386 args.add("GSDL3HOME=" + this.gsdl3home);
387 args.add("GSDL3SRCHOME=" + srcHome);
388 args.add("GSDLOS=" + this.gsdlos);
389 args.add("GSDL-RUN-SETUP=true");
390 args.add("PERL_PERTURB_KEYS=0");
391
392 if(envvars != null) {
393 for(int i = 0; i < envvars.length; i++) {
394 args.add(envvars[i]);
395 }
396 }
397
398 for (String a : System.getenv().keySet())
399 {
400 args.add(a + "=" + System.getenv(a));
401 }
402
403 String command_str = "";
404 for (int i = 0; i < command.length; i++)
405 {
406 command_str = command_str + command[i] + " ";
407 }
408
409 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = " + command_str));
410 Process prcs = null;
411 BufferedReader ebr = null;
412 BufferedReader stdinbr = null;
413 try
414 {
415 Runtime rt = Runtime.getRuntime();
416 sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
417 prcs = (dir == null)
418 ? rt.exec(command, args.toArray(new String[args.size()]))
419 : rt.exec(command, args.toArray(new String[args.size()]), dir);
420
421 InputStreamReader eisr = new InputStreamReader(prcs.getErrorStream());
422 InputStreamReader stdinisr = new InputStreamReader(prcs.getInputStream());
423 ebr = new BufferedReader(eisr);
424 stdinbr = new BufferedReader(stdinisr);
425 // Captures the std err of a program and pipes it into
426 // std in of java
427
428 File logDir = new File(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log");
429 if (!logDir.exists())
430 {
431 logDir.mkdir();
432 }
433
434 BufferedWriter bw = new BufferedWriter(new FileWriter(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log" + File.separator + "build_log." + (System.currentTimeMillis()) + ".txt"));
435 bw.write("Document Editor Build \n");
436
437 bw.write("Command = " + command_str + "\n");
438
439 String eline = null;
440 String stdinline = null;
441 while (((eline = ebr.readLine()) != null || (stdinline = stdinbr.readLine()) != null) && !this.cancel)
442 {
443 if (eline != null)
444 {
445 //System.err.println("ERROR: " + eline);
446 bw.write(eline + "\n");
447 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, eline));
448 }
449 if (stdinline != null)
450 {
451 //System.err.println("OUT: " + stdinline);
452 bw.write(stdinline + "\n");
453 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, stdinline));
454 }
455 }
456 closeResource(bw);
457
458 if (!this.cancel)
459 {
460 // Now display final message based on exit value
461 prcs.waitFor();
462
463 if (prcs.exitValue() == 0)
464 {
465 //status = OK;
466 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
467
468 success = true;
469 }
470 else
471 {
472 //status = ERROR;
473 sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));
474
475 //return false;
476 success = false;
477
478 }
479 }
480 else
481 {
482 // I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine.
483 sendProcessStatus(new ConstructionEvent(this, GSStatus.HALTED, "killing the process"));
484 //prcs.getOutputStream().close();
485 //prcs.destroy();
486 ////status = ERROR;
487
488 //return false;
489 success = false;
490 }
491 }
492 catch (Exception e)
493 {
494 e.printStackTrace();
495 sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred " + e.toString()));
496 } finally {
497 // http://steveliles.github.io/invoking_processes_from_java.html
498 // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
499 // http://mark.koli.ch/leaky-pipes-remember-to-close-your-streams-when-using-javas-runtimegetruntimeexec
500
501 if( prcs != null ) {
502 closeResource(prcs.getErrorStream());
503 closeResource(prcs.getOutputStream());
504 closeResource(prcs.getInputStream());
505 prcs.destroy();
506 }
507
508 closeResource(ebr);
509 closeResource(stdinbr);
510 }
511
512 // we're done, but we don't send a process complete message here cos there might be stuff to do after this has finished.
513 //return true;
514 return success;
515 }
516
517 public static void closeResource(Closeable resourceHandle) {
518 try {
519 if(resourceHandle != null) {
520 resourceHandle.close();
521 resourceHandle = null;
522 }
523 } catch(Exception e) {
524 System.err.println("Exception closing resource: " + e.getMessage());
525 e.printStackTrace();
526 }
527 }
528}
Note: See TracBrowser for help on using the repository browser.