source: gsdl/trunk/perllib/g2futil.pm@ 16407

Last change on this file since 16407 was 16407, checked in by ak19, 16 years ago

Corrected change made yesterday: dealing with the undefined case (as happens for GS2) when temporarily switching CATAlINA_HOME to fedora's tomcat which is necessary for GS3

File size: 12.1 KB
Line 
1package g2futil;
2
3
4BEGIN
5{
6 if (!defined $ENV{'FEDORA_HOME'}) {
7 print STDERR "Error: Environment variable FEDORA_HOME not set.\n";
8 exit 1;
9 }
10
11 my $fedora_client_bin = &util::filename_cat($ENV{'FEDORA_HOME'},"client","bin");
12 &util::envvar_append("PATH",$fedora_client_bin);
13}
14
15use strict;
16use util;
17
18sub run_cmd_old
19{
20 my ($cmd,$verbosity,$tolerate_error) = @_;
21
22 if (($verbosity == 0)
23 || (defined $tolerate_error && ($tolerate_error eq "tolerate_error"))) {
24 $cmd .= " > /dev/null"; # Too Unix specific?
25 }
26
27 if ($verbosity >= 2) {
28 print "Runing command:\n";
29 print "$cmd\n";
30 }
31
32 my $status = system($cmd);
33
34 if ($verbosity >= 2) {
35 print "Exit status = ", $status/256, "\n";
36 }
37
38 if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error")) {
39 if ($status>0) {
40 print STDERR "Error executing:\n$cmd\n";
41 print STDERR "$!\n";
42 }
43 }
44
45 return $status;
46}
47
48
49sub run_cmd
50{
51 my ($prog,$arguments,$verbosity,$tolerate_error) = @_;
52
53 my $cmd_status = undef;
54
55 my $script_ext = ($ENV{'GSDLOS'} =~ m/^windows/) ? ".bat" : ".sh";
56
57 if ($prog =~ m/^fedora-/) {
58 $prog .= $script_ext;
59 }
60 if (($prog =~ m/.pl$/i) && ($ENV{'GSDLOS'} =~ m/^windows/)) {
61 $prog ="perl -S $prog";
62 }
63
64 my $cmd = "$prog $arguments";
65
66### print "*** cmd = $cmd\n";
67
68 if (open(CMD,"$cmd 2>&1 |"))
69 {
70 my $result = "";
71 my $line;
72 while (defined ($line = <CMD>))
73 {
74 $result .= $line;
75
76 if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error"))
77 {
78 print $line;
79 }
80
81
82 }
83
84 close(CMD);
85
86 $cmd_status = $?;
87
88 if ($cmd_status == 0) {
89 # Check for any lines in result begining 'Error:'
90
91 if ($result =~ m/^Error\s*:/m) {
92 # Fedora script generated an error, but did not exit
93 # with an error status => artificially raise one
94
95 $cmd_status = -1;
96 }
97 }
98
99 if ($cmd_status != 0) {
100
101 if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error"))
102 {
103 print STDERR "Error: processing command failed. Exit status $cmd_status\n";
104
105 if ($verbosity >= 2) {
106 print STDERR " Command was: $cmd\n";
107 }
108 if ($verbosity >= 3) {
109 print STDERR "result: $result\n";
110 }
111
112 }
113 }
114 }
115 else
116 {
117 print STDERR "Error: failed to execute $cmd\n";
118 }
119
120
121 return $cmd_status;
122}
123
124
125sub run_datastore_info
126{
127 my ($pid,$options) = @_;
128
129 my $verbosity = $options->{'verbosity'};
130
131 my $hostname = $options->{'hostname'};
132 my $port = $options->{'port'};
133 my $username = $options->{'username'};
134 my $password = $options->{'password'};
135 my $protocol = $options->{'protocol'};
136
137 my $prog = "fedora-dsinfo";
138 my $arguments = "$hostname $port $username $password $pid $protocol";
139 my $status = run_cmd($prog,$arguments,$verbosity,"tolerate_error");
140
141 return $status;
142}
143
144sub run_purge
145{
146 my ($pid,$options) = @_;
147
148 my $verbosity = $options->{'verbosity'};
149
150 my $hostname = $options->{'hostname'};
151 my $port = $options->{'port'};
152 my $username = $options->{'username'};
153 my $password = $options->{'password'};
154 my $protocol = $options->{'protocol'};
155
156 my $server = "$hostname:$port";
157
158 my $prog = "fedora-purge";
159 my $arguments = "$server $username $password $pid $protocol";
160 $arguments .= " \\\n \"Automated_purge_by_g2f_script\"";
161
162 my $status = run_cmd($prog,$arguments,$verbosity);
163
164 return $status;
165}
166
167sub run_ingest
168{
169 my ($docmets_filename,$options) = @_;
170
171 my $verbosity = $options->{'verbosity'};
172
173 my $hostname = $options->{'hostname'};
174 my $port = $options->{'port'};
175 my $username = $options->{'username'};
176 my $password = $options->{'password'};
177 my $protocol = $options->{'protocol'};
178
179 my $server = "$hostname:$port";
180
181 my $prog = "fedora-ingest";
182
183 my $type = undef;
184
185 if ($ENV{'FEDORA2_HOME'}) {
186 $type = "metslikefedora1";
187 }
188 else {
189 $type = "info:fedora/fedora-system:METSFedoraExt-1.1";
190 }
191
192 my $arguments = "file \"$docmets_filename\" $type $server $username $password $protocol";
193 $arguments .= " \\\n \"Automated_purge_by_g2f_script\"";
194
195 my $status = run_cmd($prog,$arguments,$verbosity);
196
197 return $status;
198}
199
200
201sub rec_get_all_hash_dirs
202{
203 my ($full_dir,$all_dirs) = @_;
204
205 if (opendir(DIR, $full_dir)) {
206 my @sub_dirs = grep { ($_ !~ /^\./) && (-d &util::filename_cat($full_dir,$_)) } readdir(DIR);
207 closedir DIR;
208
209 my @hash_dirs = grep { $_ =~ m/\.dir$/ } @sub_dirs;
210 my @rec_dirs = grep { $_ !~ m/\.dir$/ } @sub_dirs;
211
212 foreach my $hd (@hash_dirs) {
213 my $full_hash_dir = &util::filename_cat($full_dir,$hd);
214 push(@$all_dirs,$full_hash_dir);
215 }
216
217 foreach my $rd (@rec_dirs) {
218 my $full_rec_dir = &util::filename_cat($full_dir,$rd);
219 rec_get_all_hash_dirs($full_rec_dir,$all_dirs);
220 }
221 }
222}
223
224sub get_all_hash_dirs
225{
226 my ($start_dir,$maxdocs) = @_;
227
228 my @all_dirs = ();
229 rec_get_all_hash_dirs($start_dir,\@all_dirs);
230
231 if ((defined $maxdocs) && ($maxdocs ne "")) {
232 my @maxdoc_dirs = ();
233 for (my $i=0; $i<$maxdocs; $i++) {
234 push(@maxdoc_dirs,shift(@all_dirs));
235 }
236 @all_dirs = @maxdoc_dirs;
237 }
238
239 return @all_dirs;
240}
241
242sub get_hash_id
243{
244 my ($hash_dir) = @_;
245
246 my $hash_id = undef;
247
248 my $docmets_filename = &util::filename_cat($hash_dir,"docmets.xml");
249
250 if (open(DIN,"<$docmets_filename"))
251 {
252 while (defined (my $line = <DIN>))
253 {
254 if ($line =~ m/<dc:identifier>(.*?)<\/dc:identifier>/)
255 {
256 $hash_id = $1;
257 last;
258 }
259 }
260
261 close(DIN);
262 }
263 else
264 {
265 print STDERR "Warning: Unable to open \"$docmets_filename\"\n";
266 }
267
268 return $hash_id;
269
270}
271
272
273# Subroutine to write the gsdl.xml file in FEDORA_HOME/tomcat/conf/Catalina/<host/localhost>/
274# This xml file will tell Fedora where to find the parent folder of the GS collect dir
275# so that it can obtain the FedoraMETS files for ingestion.
276# It depends on the Fedora server being on the same machine as the Greenstone server that
277# this code is part of.
278sub write_gsdl_xml_file
279{
280 my ($fedora_host, $collect_dir, $options) = @_;
281 my $verbosity = $options->{'verbosity'};
282 my $hostname = $options->{'hostname'};
283 my $port = $options->{'port'};
284 my $protocol = $options->{'protocol'};
285
286 print STDERR "Ensuring that a correct gsdl.xml file exists on the Fedora server end\n";
287 # The top of this file has already made sure that FEDORA_HOME is set, but for GS3
288 # CATALINA_HOME is set to GS' own tomcat. Since we'll be working with fedora, we need
289 # to temporarily set CATALINA_HOME to fedora's tomcat. (Catalina is undefined for GS2).
290 my $gs_catalina_home = $ENV{'CATALINA_HOME'} if defined $ENV{'CATALINA_HOME'};
291 $ENV{'CATALINA_HOME'} = &util::filename_cat($ENV{'FEDORA_HOME'}, "tomcat");
292
293 # 1. Find out which folder to write to: fedora_host or localhost
294 # whichever contains fedora.xml is the one we want (if none, exit with error value?)
295 my $fedora_home = $ENV{'FEDORA_HOME'};
296 my $base_path = &util::filename_cat($fedora_home, "tomcat", "conf", "Catalina");
297
298 my $host_path = &util::filename_cat($base_path, $fedora_host);
299 my $xmlFile = &util::filename_cat($host_path, "fedora.xml");
300 if (!-e $xmlFile) {
301 # check if the folder localhost contains fedoraXML
302 $host_path = &util::filename_cat($base_path, "localhost");
303 $xmlFile = &util::filename_cat($host_path, "fedora.xml");
304 if(!-e $xmlFile) {
305 # try putting gsdl in this folder, but still print a warning
306 print STDERR "$host_path does not contain file fedora.xml. Hoping gsdl.xml belongs there anyway\n";
307 }
308 }
309
310 # 2. Construct the string we are going write to the gsdl.xml file
311 # a. get the parent directory of collect_dir by removinbg the word
312 # "collect" from it and any optional OS-type slash at the end.
313 # (Path slash direction does not matter here.)
314 my $collectParentDir = $collect_dir;
315 $collectParentDir =~ s/collect(\/|\\)?//;
316
317 # b. Use the collectParentDir to create the contents of gsdl.xml
318 my $gsdlXMLcontents = "<?xml version='1.0' encoding='utf-8'?>\n<Context docBase=\"";
319 $gsdlXMLcontents = $gsdlXMLcontents.$collectParentDir."\" path=\"/gsdl\"></Context>";
320
321 # 3. If there is already a gsdl.xml file in host_path, compare the string we
322 # want to write with what is already in there. If they're the same, we can return
323 $xmlFile = &util::filename_cat($host_path, "gsdl.xml");
324 if(-e $xmlFile) {
325 # such a file exists, so read the contents
326 unless(open(FIN, "<$xmlFile")) {
327 print STDERR "g2f-import.pl: Unable to open existing $xmlFile for comparing...Recoverable. $!\n";
328 # doesn't matter, we'll just overwrite it then
329 }
330 my $xml_contents;
331 {
332 local $/ = undef; # Read entire file at once
333 $xml_contents = <FIN>; # Now file is read in as one single 'line'
334 }
335 close(FIN); # close the file
336 if($xml_contents eq $gsdlXMLcontents) {
337 print STDERR "Fedora links to the FLI import folder through gsdl.xml.\n";
338 # it already contains what we want, we're done
339 return "gsdl.xml";
340 }
341 }
342
343 # 4. If we're here, the contents of gsdl.xml need to be updated:
344 # a. First stop the fedora server
345 my $script_ext = ($ENV{'GSDLOS'} =~ m/^windows/) ? ".bat" : ".sh";
346 my $stop_tomcat = &util::filename_cat($fedora_home, "tomcat", "bin", "shutdown".$script_ext);
347 # execute the command
348 $! = 0; # does this initialise the return value?
349 my $status = system($stop_tomcat);
350 if ($status!=0) { # to get the actual exit value, divide by 256, but not useful here
351 # possible tomcat was already stopped - it's not the end of the world
352 print STDERR "Failed to stop Fedora server. Perhaps it was not running. $!\n";
353 print "Exit status = ", $status/256, "\n";
354 }
355
356 # b. overwrite the file that has outdated contents with the contents we just constructed
357 unless(open(FOUT, ">$xmlFile")) { # create or overwrite gsdl.xml file
358 die "g2f-import.pl: Unable to open $xmlFile for telling Fedora where the collect dir is...ERROR: $!\n";
359 }
360 # write out the updated contents and close the file
361 print FOUT $gsdlXMLcontents;
362 close(FOUT);
363
364 # c. Restart the fedora server
365 my $start_tomcat = &util::filename_cat($fedora_home, "tomcat", "bin", "startup".$script_ext);
366 $! = 0;
367 $status = system($start_tomcat);
368 if ($status!=0) {
369 print STDERR "Failed to restart the Fedora server... ERROR: $!\n";
370 print "Exit status = ", $status/256, "\n";
371 }
372
373 # reset CATALINA_HOME to GS' Tomcat (it is undefined for GS2 since GS2 has no tomcat):
374 $ENV{'CATALINA_HOME'} = $gs_catalina_home if defined $gs_catalina_home;
375
376 # Starting up the Fedora server takes a long time. We need to wait for the server to be
377 # ready before import can continue, because g2f-import relies on an up-and-running Fedora
378 # server to purge the collection from it while g2f-build.pl needs a ready Fedora server
379 # in order to make it ingest the FedoraMETS. Sleeping is not sufficient (#sleep 10;) since
380 # the subsequent steps depend on a proper server restart.
381 # Dr Bainbridge's suggestion: test the server is ready with a call to wget.
382
383 # Wget tries to retrieve the fedora search page (protocol://host:port/fedora/search)
384 # 20 times, waiting 3 seconds between each failed attempt. If it ultimately fails, we
385 # print a message to the user.
386 # The wget --spider option makes it check that the page is merely there rather than
387 # downloading it (see http://www.gnu.org/software/wget/manual/wget.html#Download-Options)
388 # -q is for quiet, --tries for the number of retries, --waitretry is the number of seconds
389 # between each attempt. Usually wget returns the contents of the page, but in our case it
390 # will return 0 for success since we are not downloading.
391
392 print STDERR "Fedora server restarted. Waiting for it to become ready...\n";
393 #print STDERR "****$protocol://$hostname:$port/fedora/search\n";
394
395 $! = 0;
396 my $fedoraServerReady = system("wget -q --spider --waitretry=3 --tries=20 $protocol://$hostname:$port/fedora/search");
397 if($fedoraServerReady != 0) {
398 print STDERR "Fedora server is still not ready... ERROR: $!\n";
399 print "Exit status = ", $fedoraServerReady/256, "\n";
400 die "Exiting....\n";
401 }
402
403 # return some indication that things went well
404 return "gsdl.xml";
405}
406
407
4081;
Note: See TracBrowser for help on using the repository browser.