source: main/trunk/greenstone3/src/java/org/greenstone/gsdl3/build/GS2PerlConstructor.java@ 29947

Last change on this file since 29947 was 29947, checked in by ak19, 9 years ago

First of two commits to fix rebuilding a collection using the online editor on Windows. Resolves the file lock problem. The java code works with changes in activate.pl. activate.pl is passed a flag so that it no longer de-activates and re-activates the collection itself, but just concerns itself with moving building to index. The GS2Construct java code now de-activates the collection prior to calling activate.pl and then re-activates it afterward. In the way it was done before, activate.pl used to handle de- and re-activating the collection. But when it was launched from the server java code, the java VM would exit having left a copy of the file handles to the perl process when forking the process for activate.pl. The perl code could not move building to index since the file handles had locks (6 of them) on the index/text/collection.gdb. Changes have been made to GS2PerlConstructor too, so that it more cleanly closes all the pipes of a process, that the process itself may thereby exit cleanly. Not yet able to move this properly into its own classes since the StreamGobbler classes in GLI are not quite suited but were customised for FormatConverter.

  • Property svn:keywords set to Author Date Id Revision
File size: 16.2 KB
Line 
1package org.greenstone.gsdl3.build;
2
3// greenstome classes
4import org.greenstone.gsdl3.util.*;
5import org.greenstone.util.Misc;
6import org.greenstone.util.GlobalProperties;
7
8// xml classes
9import org.w3c.dom.Element;
10import org.w3c.dom.NodeList;
11
12//general java classes
13import java.io.BufferedReader;
14import java.io.BufferedWriter;
15import java.io.Closeable;
16import java.io.FileWriter;
17import java.io.InputStreamReader;
18import java.io.File;
19import java.io.IOException;
20import java.util.ArrayList;
21import java.util.Vector;
22
23import org.apache.log4j.*;
24
25/**
26 * CollectionConstructor class for greenstone 2 compatible building it uses the
27 * perl scripts to do the building stuff
28 */
29public class GS2PerlConstructor extends CollectionConstructor
30{
31 static Logger logger = Logger.getLogger(org.greenstone.gsdl3.build.GS2PerlConstructor.class.getName());
32
33 public static final int NEW = 0;
34 public static final int IMPORT = 1;
35 public static final int BUILD = 2;
36 public static final int ACTIVATE = 3;
37 public static final int SET_METADATA_SERVER = 4;
38
39 /**
40 * gsdlhome for greenstone 2 - we use the perl modules and building scripts
41 * from there
42 */
43 protected String gsdl2home = null;
44 /** gsdlhome for gsdl3 - shouldn't need this eventually ?? */
45 protected String gsdl3home = null;
46 /** gsdlos for greenstone 2 */
47 protected String gsdlos = null;
48 /** the path environment variable */
49 protected String path = null;
50
51 public GS2PerlConstructor(String name)
52 {
53 super(name);
54 }
55
56 /** retrieves the necessary environment variables */
57 public boolean configure()
58 {
59 // try to get the environment variables
60 this.gsdl3home = GlobalProperties.getGSDL3Home();
61 this.gsdl2home = this.gsdl3home + File.separator + ".." + File.separator + "gs2build";
62 this.gsdlos = Misc.getGsdlOS();
63
64 this.path = System.getenv("PATH");
65
66 if (this.gsdl2home == null)
67 {
68 System.err.println("You must have gs2build installed, and GSDLHOME set for GS2Perl building to work!!");
69 return false;
70 }
71 if (this.gsdl3home == null || this.gsdlos == null)
72 {
73 System.err.println("You must have GSDL3HOME and GSDLOS set for GS2Perl building to work!!");
74 return false;
75 }
76 if (this.path == null)
77 {
78 System.err.println("You must have the PATH set for GS2Perl building to work!!");
79 return false;
80 }
81 return true;
82 }
83
84 public void run()
85 {
86 String msg;
87 ConstructionEvent evt;
88 if (this.process_type == -1)
89 {
90 msg = "Error: you must set the action type";
91 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
92 sendMessage(evt);
93 return;
94 }
95 if (this.site_home == null)
96 {
97 msg = "Error: you must set site_home";
98 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
99 sendMessage(evt);
100 return;
101 }
102 if (this.process_type != NEW && this.collection_name == null)
103 {
104 msg = "Error: you must set collection_name";
105 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
106 sendMessage(evt);
107 return;
108 }
109
110 switch (this.process_type)
111 {
112 case NEW:
113 newCollection();
114 break;
115 case IMPORT:
116 importCollection();
117 break;
118 case BUILD:
119 buildCollection();
120 break;
121 case ACTIVATE:
122 activateCollection();
123 break;
124 case SET_METADATA_SERVER:
125 setMetadataForCollection();
126 break;
127 default:
128 msg = "wrong type of action specified!";
129 evt = new ConstructionEvent(this, GSStatus.ERROR, msg);
130 sendMessage(evt);
131 break;
132 }
133 }
134
135 protected void newCollection()
136 {
137 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: new collection."));
138 Vector<String> command = new Vector<String>();
139 command.add("gs2_mkcol.pl");
140 command.add("-site");
141 command.add(this.site_home);
142 command.add("-collectdir");
143 command.add(GSFile.collectDir(this.site_home));
144 command.addAll(extractParameters(this.process_params));
145 command.add(this.collection_name);
146 String[] command_str = {};
147 command_str = command.toArray(command_str);
148 if (runPerlCommand(command_str))
149 {
150 // success!! - need to send the final completed message
151 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
152 } // else an error message has already been sent, do nothing
153
154 }
155
156 protected void importCollection()
157 {
158 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: import collection."));
159 Vector<String> command = new Vector<String>();
160
161 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
162 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
163 {
164 perlPath = perlPath + File.separator;
165 }
166
167 command.add(perlPath + "perl");
168 command.add("-S");
169 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "import.pl");
170 if (this.manifest_file != null)
171 {
172 command.add("-keepold");
173 command.add("-manifest");
174 command.add(this.manifest_file);
175 }
176 command.add("-site");
177 command.add(this.site_name);
178 command.add("-collectdir");
179 command.add(GSFile.collectDir(this.site_home));
180 command.addAll(extractParameters(this.process_params));
181 command.add(this.collection_name);
182 String[] command_str = {};
183 command_str = command.toArray(command_str);
184
185 if (runPerlCommand(command_str))
186 {
187 // success!! - need to send the final completed message
188 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
189 } // else an error message has already been sent, do nothing
190 }
191
192 protected void buildCollection()
193 {
194 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: build collection."));
195 Vector<String> command = new Vector<String>();
196
197 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
198 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
199 {
200 perlPath = perlPath + File.separator;
201 }
202
203 command.add(perlPath + "perl");
204 command.add("-S");
205 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "buildcol.pl");
206 command.add("-site");
207 command.add(this.site_name);
208 command.add("-collectdir");
209 command.add(GSFile.collectDir(this.site_home));
210 command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly
211 command.addAll(extractParameters(this.process_params));
212 command.add(this.collection_name);
213
214 String[] command_str = {};
215 command_str = command.toArray(command_str);
216
217 if (runPerlCommand(command_str))
218 {
219 // success!! - need to send the final completed message
220 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
221 }// else an error message has already been sent, do nothing
222 }
223
224 protected void activateCollection()
225 {
226 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection construction: activate collection."));
227
228 // first check that we have a building directory
229 // (don't want to bother running activate.pl otherwise)
230 File build_dir = new File(GSFile.collectionBuildDir(this.site_home, this.collection_name));
231 if (!build_dir.exists())
232 {
233 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "build dir doesn't exist!"));
234 return;
235 }
236
237 /*
238
239 // move building to index
240 File index_dir = new File(GSFile.collectionIndexDir(this.site_home, this.collection_name));
241 if (index_dir.exists())
242 {
243 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "deleting index directory"));
244 GSFile.deleteFile(index_dir);
245 if (index_dir.exists())
246 {
247 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index directory still exists!"));
248 return;
249 }
250 }
251
252 GSFile.moveDirectory(build_dir, index_dir);
253 if (!index_dir.exists())
254 {
255 sendMessage(new ConstructionEvent(this, GSStatus.ERROR, "index dir wasn't created!"));
256 }
257
258 // success!! - need to send the final completed message
259 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
260 */
261
262 // Running activate.pl instead of making java move building to index as above
263 // circumvents the issue of the jdbm .lg log file (managed by TransactionManager)
264 // in index dir not getting deleted at times. The perl code is able to delete this
265 // sucessfully consistently during testing, whereas java at times is unable to delete it.
266 Vector<String> command = new Vector<String>();
267
268 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
269 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
270 {
271 perlPath = perlPath + File.separator;
272 }
273
274 command.add(perlPath + "perl");
275 command.add("-S");
276 command.add(GlobalProperties.getGS2Build() + File.separator + "bin" + File.separator + "script" + File.separator + "activate.pl");
277 command.add("-site");
278 command.add(this.site_name);
279 command.add("-collectdir");
280 command.add(GSFile.collectDir(this.site_home));
281 command.add("-removeold"); // saves some seconds processing time when this flag's added in explicitly
282 command.add("-skipactivation"); // gsdl3/util/GS2Construct does the activation and reactivation
283 command.addAll(extractParameters(this.process_params));
284 command.add(this.collection_name);
285
286 String[] command_str = {};
287 command_str = command.toArray(command_str);
288
289 if (runPerlCommand(command_str))
290 {
291 // success!! - need to send the final completed message
292 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
293 }// else an error message has already been sent, do nothing
294
295 }
296
297
298 protected void setMetadataForCollection()
299 {
300 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "Collection metadata: setMetadata for collection."));
301
302 Vector<String> command = new Vector<String>();
303
304 String perlPath = GlobalProperties.getProperty("perl.path", "perl");
305 if (perlPath.charAt(perlPath.length() - 1) != File.separatorChar)
306 {
307 perlPath = perlPath + File.separator;
308 }
309
310 String cgi_directory = GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi";
311 command.add(perlPath + "perl");
312 command.add("-S");
313 //command.add(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi" + File.separator + "metadata-server.pl");
314 command.add(cgi_directory + File.separator + "metadata-server.pl");
315
316 // Need to set QUERY_STRING and REQUEST_METHOD=GET in environment
317 // http://www.cgi101.com/class/ch3/text.html
318 String[] envvars = {
319 "QUERY_STRING=" + this.query_string,
320 "REQUEST_METHOD=GET"
321 };
322
323 String[] command_str = {};
324 command_str = command.toArray(command_str);
325
326 // http://www.cgi101.com/class/ch3/text.html
327 // setenv QUERY_STRING and REQUEST_METHOD = GET.
328 if (runPerlCommand(command_str, envvars, new File(cgi_directory)))
329 //new File(GlobalProperties.getGSDL3Home() + File.separator + "WEB-INF" + File.separator + "cgi")))
330 {
331 // success!! - need to send the final completed message
332 sendProcessComplete(new ConstructionEvent(this, GSStatus.COMPLETED, ""));
333 }// else an error message has already been sent, do nothing
334
335 }
336
337 /** extracts all the args from the xml and returns them in a Vector */
338 protected Vector<String> extractParameters(Element param_list)
339 {
340
341 Vector<String> args = new Vector<String>();
342 if (param_list == null)
343 {
344 return args; // return an empty vector
345 }
346 NodeList params = param_list.getElementsByTagName(GSXML.PARAM_ELEM);
347
348 for (int i = 0; i < params.getLength(); i++)
349 {
350 Element p = (Element) params.item(i);
351 String name = p.getAttribute(GSXML.NAME_ATT);
352 String value = p.getAttribute(GSXML.VALUE_ATT);
353 if (!name.equals(""))
354 {
355 args.add("-" + name);
356 if (!value.equals(""))
357 {
358 args.add(value);
359 }
360 }
361 }
362
363 return args;
364 }
365
366 /** returns true if completed correctly, false otherwise */
367 protected boolean runPerlCommand(String[] command) {
368 return runPerlCommand(command, null, null);
369 }
370
371 protected boolean runPerlCommand(String[] command, String[] envvars, File dir)
372 {
373 boolean success = true;
374
375 int sepIndex = this.gsdl3home.lastIndexOf(File.separator);
376 String srcHome = this.gsdl3home.substring(0, sepIndex);
377
378 ArrayList<String> args = new ArrayList<String>();
379 args.add("GSDLHOME=" + this.gsdl2home);
380 args.add("GSDL3HOME=" + this.gsdl3home);
381 args.add("GSDL3SRCHOME=" + srcHome);
382 args.add("GSDLOS=" + this.gsdlos);
383 args.add("GSDL-RUN-SETUP=true");
384 args.add("PERL_PERTURB_KEYS=0");
385
386 if(envvars != null) {
387 for(int i = 0; i < envvars.length; i++) {
388 args.add(envvars[i]);
389 }
390 }
391
392 for (String a : System.getenv().keySet())
393 {
394 args.add(a + "=" + System.getenv(a));
395 }
396
397 String command_str = "";
398 for (int i = 0; i < command.length; i++)
399 {
400 command_str = command_str + command[i] + " ";
401 }
402
403 sendMessage(new ConstructionEvent(this, GSStatus.INFO, "command = " + command_str));
404 Process prcs = null;
405 BufferedReader ebr = null;
406 BufferedReader stdinbr = null;
407 try
408 {
409 Runtime rt = Runtime.getRuntime();
410 sendProcessBegun(new ConstructionEvent(this, GSStatus.ACCEPTED, "starting"));
411 prcs = (dir == null)
412 ? rt.exec(command, args.toArray(new String[args.size()]))
413 : rt.exec(command, args.toArray(new String[args.size()]), dir);
414
415 InputStreamReader eisr = new InputStreamReader(prcs.getErrorStream());
416 InputStreamReader stdinisr = new InputStreamReader(prcs.getInputStream());
417 ebr = new BufferedReader(eisr);
418 stdinbr = new BufferedReader(stdinisr);
419 // Captures the std err of a program and pipes it into
420 // std in of java
421
422 File logDir = new File(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log");
423 if (!logDir.exists())
424 {
425 logDir.mkdir();
426 }
427
428 BufferedWriter bw = new BufferedWriter(new FileWriter(GSFile.collectDir(this.site_home) + File.separator + this.collection_name + File.separator + "log" + File.separator + "build_log." + (System.currentTimeMillis()) + ".txt"));
429 bw.write("Document Editor Build \n");
430
431 bw.write("Command = " + command_str + "\n");
432
433 String eline = null;
434 String stdinline = null;
435 while (((eline = ebr.readLine()) != null || (stdinline = stdinbr.readLine()) != null) && !this.cancel)
436 {
437 if (eline != null)
438 {
439 //System.err.println("ERROR: " + eline);
440 bw.write(eline + "\n");
441 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, eline));
442 }
443 if (stdinline != null)
444 {
445 //System.err.println("OUT: " + stdinline);
446 bw.write(stdinline + "\n");
447 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, stdinline));
448 }
449 }
450 closeResource(bw);
451
452 if (!this.cancel)
453 {
454 // Now display final message based on exit value
455 prcs.waitFor();
456
457 if (prcs.exitValue() == 0)
458 {
459 //status = OK;
460 sendProcessStatus(new ConstructionEvent(this, GSStatus.CONTINUING, "Success"));
461
462 success = true;
463 }
464 else
465 {
466 //status = ERROR;
467 sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Failure"));
468
469 //return false;
470 success = false;
471
472 }
473 }
474 else
475 {
476 // I need to somehow kill the child process. Unfortunately Thread.stop() and Process.destroy() both fail to do this. But now, thankx to the magic of Michaels 'close the stream suggestion', it works fine.
477 sendProcessStatus(new ConstructionEvent(this, GSStatus.HALTED, "killing the process"));
478 //prcs.getOutputStream().close();
479 //prcs.destroy();
480 ////status = ERROR;
481
482 //return false;
483 success = false;
484 }
485 }
486 catch (Exception e)
487 {
488 e.printStackTrace();
489 sendProcessStatus(new ConstructionEvent(this, GSStatus.ERROR, "Exception occurred " + e.toString()));
490 } finally {
491 // http://steveliles.github.io/invoking_processes_from_java.html
492 // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
493 // http://mark.koli.ch/leaky-pipes-remember-to-close-your-streams-when-using-javas-runtimegetruntimeexec
494
495 if( prcs != null ) {
496 closeResource(prcs.getErrorStream());
497 closeResource(prcs.getOutputStream());
498 closeResource(prcs.getInputStream());
499 prcs.destroy();
500 }
501
502 closeResource(ebr);
503 closeResource(stdinbr);
504 }
505
506 // we're done, but we don't send a process complete message here cos there might be stuff to do after this has finished.
507 //return true;
508 return success;
509 }
510
511 public static void closeResource(Closeable resourceHandle) {
512 try {
513 if(resourceHandle != null) {
514 resourceHandle.close();
515 resourceHandle = null;
516 }
517 } catch(Exception e) {
518 System.err.println("Exception closing resource: " + e.getMessage());
519 e.printStackTrace();
520 }
521 }
522}
Note: See TracBrowser for help on using the repository browser.