source: gs3-extensions/solr/trunk/src/perllib/solrserver.pm@ 29349

Last change on this file since 29349 was 29349, checked in by ak19, 10 years ago

Fixing the problem on Mac Lion binaries (not svn) of wget not running when solrserver.pm invokes it, which results in rebuilding solr collections from GLI or from the cmd line to fail. If the gnome-lib env is set up, then the gs2build/bin/darwin/wget executable of Lion binaries works. Don't want to adjust the overall environment of the terminal, as that can mess up display of graphical programs like emacs at least on linux. Instead, calling util::set_gnomelib_env() that was introduced earlier this year before running the wget command. Doing so sets the gnome-lib env for the wget cmd. This is the solution that was since recently already being used in gs2build/bin/script/gsWget.pl

File size: 15.3 KB
Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr/jetty server
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27package solrserver;
28
29use strict;
30#no strict 'refs';
31
32use solrutil;
33
34sub new {
35 my $class = shift(@_);
36 my ($build_dir) = @_;
37
38 my $self = { 'jetty_stop_key' => "greenstone-solr" };
39
40 $self->{'build_dir'} = $build_dir;
41
42 my $search_path = &solrutil::get_search_path();
43
44 my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
45 my $full_server_jar = solrutil::locate_file($search_path,$server_jar);
46 $self->{'full_server_jar'} = $full_server_jar;
47
48 $self->{'jetty_explicitly_started'} = undef;
49
50 my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
51 my $base_url = "http://localhost:$jetty_server_port/solr/";
52 my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
53
54 $self->{'base-url'} = $base_url;
55 $self->{'admin-url'} = $admin_url;
56
57 return bless $self, $class;
58}
59
60sub set_jetty_stop_key {
61 my $self = shift (@_);
62 my ($stop_key) = @_;
63
64 $self->{'jetty_stop_key'} = $stop_key if defined $stop_key;
65}
66
67sub _wget_service
68{
69 my $self = shift (@_);
70 my ($output_format,$url,$cgi_get_args) = @_;
71
72 my $full_url = $url;
73
74 $url .= "?$cgi_get_args" if (defined $cgi_get_args);
75
76## print STDERR "\n\n**** _wget_service SOLR WEB URL: $url\n\n";
77
78 # the wget binary is dependent on the gnomelib_env (particularly lib/libiconv2.dylib) being set, particularly on Mac Lion binaries (android too?)
79 &util::set_gnomelib_env(); # this will set the gnomelib env once for each subshell launched, by first checking if GEXTGNOME is not already set
80
81 my $cmd = "wget -O - \"$url\" 2>&1";
82
83 my $preamble_output = "";
84 my $xml_output = "";
85 my $error_output = undef;
86
87 my $in_preamble = ($output_format eq "xml") ? 1 : 0;
88
89## print STDERR "**** wgetcmd = \n $cmd\n";
90
91 if (open(WIN,"$cmd |")) {
92
93 my $line;
94 while (defined ($line=<WIN>)) {
95
96 if ($line =~ m/ERROR \d+:/) {
97 chomp $line;
98 $error_output = $line;
99 last;
100 }
101 elsif ($line =~ m/failed: Connection refused/) {
102 chomp $line;
103 $error_output = $line;
104 last;
105 }
106 elsif ($in_preamble) {
107 if ($line =~ m/<.*>/) {
108 $in_preamble = 0;
109 }
110 else {
111 $preamble_output .= $line;
112 }
113 }
114
115 if (! $in_preamble) {
116 $xml_output .= $line;
117 }
118 }
119 close(WIN);
120
121 }
122 else {
123 $error_output = "Error: failed to run $cmd\n";
124 $error_output .= " $!\n";
125 }
126
127 if(defined $error_output) {
128 print STDERR "\n\n**** WGET_SERVICE got an error: $error_output\n\n";
129 }
130
131 my $output = { 'url' => $full_url,
132 'preamble' => $preamble_output,
133 'output' => $xml_output,
134 'error' => $error_output };
135
136 return $output;
137}
138
139
140sub _base_service
141{
142 my $self = shift (@_);
143 my ($cgi_get_args) = @_;
144
145 my $base_url = $self->{'base-url'};
146
147 return $self->_wget_service("html",$base_url,$cgi_get_args);
148}
149
150sub _admin_service
151{
152 my $self = shift (@_);
153 my ($cgi_get_args) = @_;
154
155 my $admin_url = $self->{'admin-url'};
156
157 return $self->_wget_service("xml",$admin_url,$cgi_get_args);
158}
159
160
161sub server_running
162{
163 my $self = shift @_;
164
165 my $output = $self->_base_service();
166
167 my $have_error = defined $output->{'error'};
168
169 my $running = ($have_error) ? 0 : 1;
170
171 return $running;
172}
173
174
175sub admin_ping_core
176{
177 my $self = shift @_;
178 my ($core) = @_;
179
180 my $cgi_get_args = "action=STATUS&core=$core";
181
182 my $ping_status = 1;
183
184 my $output = $self->_admin_service($cgi_get_args);
185
186 if (defined $output->{'error'}) {
187 # severe error, such as failing to connect to the server
188 $ping_status = 0;
189
190 my $url = $output->{'url'};
191 my $preamble = $output->{'preamble'};
192 my $error = $output->{'error'};
193
194 print STDERR "----\n";
195 print STDERR "Error: Failed to get XML response from:\n";
196 print STDERR " $url\n";
197 print STDERR "Output was:\n";
198 print STDERR $preamble if ($preamble ne "");
199 print STDERR "$error\n";
200 print STDERR "----\n";
201 }
202 else {
203
204 # If the collection doesn't exist yet, then there will be
205 # an empty element of the form:
206 # <lst name="collect-doc"/>
207 # where 'collect' is the actual name of the collection,
208 # such as demo
209
210 my $xml_output = $output->{'output'};
211
212 my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
213
214 $ping_status = !($xml_output =~ m/$empty_element/s);
215 }
216
217 return $ping_status;
218}
219
220sub filtered_copy
221{
222 my $self = shift @_;
223
224 my $src_file = shift @_;
225 my $dst_file = shift @_;
226 my $re_substitutions = shift @_;
227
228 # $re_substitutions is a hashmap of the form: [re_key] => subst_str
229
230 my $content = "";
231
232 if (open(FIN,'<:utf8',$src_file)) {
233
234 my $line;
235 while (defined($line=<FIN>)) {
236 $content .= $line;
237 }
238 }
239
240 close(FIN);
241
242 # perform RE string substitutions
243 foreach my $re_key (keys %$re_substitutions) {
244
245 my $subst_str = $re_substitutions->{$re_key};
246
247 # Perform substitution of the form:
248 # $content =~ s/$re_key/$subst_str/g;
249 # but allow allow separator char (default '/')
250 # and flags (default 'g') to be parameterized
251
252 $content =~ s/$re_key/$subst_str/g;
253 }
254
255 if (open(FOUT, '>:utf8', $dst_file)) {
256 print FOUT $content;
257 close(FOUT);
258 }
259 else {
260 print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
261 }
262}
263
264sub solr_xml_to_solr_xml_in
265{
266 my $self = shift @_;
267 my ($solr_xml_dir) = @_;
268
269 my $gsdl3home = $ENV{'GSDL3HOME'};
270
271 if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
272 # if not passed in, use stored solr_live_home
273 $solr_xml_dir = $self->{'solr_live_home'};
274 }
275 my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
276 my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
277
278 my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
279
280 my $replacement_map = { qr/$gsdl3home_re/ => "\@gsdl3home\@" };
281
282 $self->filtered_copy($solrxml,$solrxml_in,$replacement_map);
283}
284
285
286sub solr_xml_in_to_solr_xml
287{
288 my $self = shift @_;
289 my ($solr_xml_dir) = @_;
290
291 my $gsdl3home = $ENV{'GSDL3HOME'};
292 if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
293 # if not passed in, use stored solr home
294 $solr_xml_dir = $self->{'solr_live_home'};
295 }
296 my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
297 my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
298
299 my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
300
301 my $replacement_map = { qr/\@gsdl3home\@/ => $gsdl3home_re };
302
303 $self->filtered_copy($solrxml_in,$solrxml,$replacement_map);
304}
305
306
307# Some of the Solr CoreAdmin API calls available.
308# See http://wiki.apache.org/solr/CoreAdmin
309sub admin_reload_core
310{
311 my $self = shift @_;
312 my ($core) = @_;
313
314 my $cgi_get_args = "action=RELOAD&core=$core";
315
316 $self->_admin_service($cgi_get_args);
317
318}
319
320sub admin_rename_core
321{
322 my $self = shift @_;
323 my ($oldcore, $newcore) = @_;
324
325 my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
326
327 $self->_admin_service($cgi_get_args);
328
329}
330
331sub admin_swap_core
332{
333 my $self = shift @_;
334 my ($oldcore, $newcore) = @_;
335
336 my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
337
338 $self->_admin_service($cgi_get_args);
339
340}
341
342# The ALIAS action is not supported in our version of solr (despite it
343# being marked as experimental in the documentation for Core Admin)
344sub admin_alias_core
345{
346 my $self = shift @_;
347 my ($oldcore, $newcore) = @_;
348
349 my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
350
351 $self->_admin_service($cgi_get_args);
352
353}
354
355sub admin_create_core
356{
357 my $self = shift @_;
358 my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
359
360 my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
361
362 my $cgi_get_args = "action=CREATE&name=$core";
363
364 my $collect_home = $ENV{'GSDLCOLLECTDIR'};
365 my $etc_dirname = &util::filename_cat($collect_home,"etc");
366
367 if(!defined $data_parent_dir) {
368 $data_parent_dir = $self->{'build_dir'};
369 }
370
371 my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir"
372
373 $cgi_get_args .= "&instanceDir=$etc_dirname";
374 $cgi_get_args .= "&dataDir=$idx_dirname";
375
376 $self->_admin_service($cgi_get_args);
377
378}
379
380# removes (unloads) core from the ext/solr/sorl.xml config file
381sub admin_unload_core
382{
383 my $self = shift @_;
384 my ($core, $delete) = @_;
385
386 my $cgi_get_args = "action=UNLOAD&core=$core"; # &deleteIndex=true from Solr3.3
387 if(defined $delete && $delete == 1) {
388 $cgi_get_args = $cgi_get_args."&deleteIndex=true";
389 }
390
391 $self->_admin_service($cgi_get_args);
392
393}
394
395sub start
396{
397 my $self = shift @_;
398 my ($verbosity) = @_;
399
400 $verbosity = 1 unless defined $verbosity;
401
402 my $solr_home = $ENV{'GEXT_SOLR'};
403 my $solr_live_home = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr");
404 my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
405 my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
406
407 chdir($solr_home);
408 #$self->{'solr_home'} = $solr_home;
409 $self->{'solr_live_home'} = $solr_live_home;
410## my $solr_etc = &util::filename_cat($solr_home,"etc");
411
412 my $server_props = "-DSTOP.PORT=$jetty_stop_port";
413 $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
414 #$server_props .= " -Dsolr.solr.home=$solr_home";
415$server_props .= " -Dsolr.solr.home=$solr_live_home";
416 my $full_server_jar = $self->{'full_server_jar'};
417
418 # https://wiki.apache.org/solr/SolrLogging
419 my $solr_slf4j = &util::filename_cat($solr_home, "lib", "ext", "*");
420 my $solr_lib_java = &util::filename_cat($solr_home, "lib", "java", "*");
421 my $solr_jar_dep = &util::pathname_cat($solr_lib_java, $solr_slf4j);
422
423 my $solr_log4j = &util::filename_cat($solr_home, "conf", "log4j.properties");
424 if (defined $ENV{'GSDLOS'} && (($ENV{'GSDLOS'} =~ m/windows/) && ($^O ne "cygwin"))) {
425 $solr_log4j =~ s@\\@\/@g;
426 $solr_log4j = "/$solr_log4j"; # Three slashes on windows: file///c:/path/to/log4j.properties
427 }
428
429 #my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_home/lib/java/*:$solr_slf4j/*\" $server_props -jar \"$full_server_jar\"";
430 my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_jar_dep\" $server_props -jar \"$full_server_jar\"";
431 my $server_status = "unknown";
432
433 if ($self->server_running()) {
434 $server_status = "already-running";
435 }
436 elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
437
438 print STDERR "**** starting up solr jetty server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
439
440 my $line;
441 while (defined($line=<STARTIN>)) {
442 # Scan through output until you see a line like:
443 # 2011-08-22 .. :INFO::Started [email protected]:8983
444 # which signifies that the server has started up and is
445 # "ready and listening"
446
447 if (($line =~ m/^(WARN|ERROR|SEVERE):/)
448 || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
449 print "Jetty startup: $line";
450 }
451
452 if ($line =~ m/WARN.*failed SocketConnector/) {
453 if ($line =~ m/Address already in use/) {
454 $server_status = "already-running";
455 }
456 else {
457 $server_status = "failed-to-start";
458 }
459 last;
460 }
461
462 if ($line =~ m/INFO.*Started SocketConnector/) {
463 $server_status = "explicitly-started";
464 last;
465 }
466 }
467 }
468 else {
469 print STDERR "Error: failed to start solr-jetty-server\n";
470 print STDERR "$!\n";
471 print STDERR "Command attempted was:\n";
472 print STDERR " $server_java_cmd\n";
473 print STDERR "run from directory:\n";
474 print STDERR " $solr_home\n";
475 print STDERR "----\n";
476
477 exit -1;
478 }
479
480 if ($server_status eq "explicitly-started") {
481 $self->{'jetty_explicitly_started'} = 1;
482 print "Jetty server ready and listening for connections on port";
483 print " $jetty_server_port\n";
484
485 # now we know the server is ready to accept connections, fork a
486 # child process that continues to listen to the output and
487 # prints out any lines that are not INFO lines
488
489 if (fork()==0) {
490 # child process
491
492 my $line;
493 while (defined ($line = <STARTIN>)) {
494
495# if (($line =~ m/^(WARN|ERROR|SEVERE):/)
496# || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
497# print "Jetty $line";
498# }
499
500 # skip info lines
501 next if ($line =~ m/^INFO:/);
502 next if ($line =~ m/^[0-9 :-]*INFO::/);
503 next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
504 next if ($line =~ m/^\d{4}-\d{2}-\d{2}\s+/);
505
506## next if ($line =~ m/^\s*\w+{.*}/);
507
508 # if here, then some non-trival message has been logged
509 print "Jetty/Solr processing: $line";
510 }
511 close(STARTIN);
512
513 # And now stop nicely
514 exit 0;
515 }
516 # otherwise let the parent continue on
517 }
518 elsif ($server_status eq "already-running") {
519 print STDERR "Using existing server detected on port $jetty_server_port\n";
520 $self->{'jetty_explicitly_started'} = 0;
521 }
522 elsif ($server_status eq "failed-to-start") {
523 print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
524 print STDERR ", but encountered an initialization error\n";
525 exit -1;
526 }
527
528}
529
530sub explicitly_started
531{
532 my $self = shift @_;
533
534 return $self->{'jetty_explicitly_started'};
535}
536
537sub stop
538{
539 my $self = shift @_;
540 my ($options) = @_;
541
542 my $solr_home = $ENV{'GEXT_SOLR'};
543
544 chdir($solr_home);
545
546 # defaults
547 my $do_wait = 1;
548 my $output_verbosity = 1;
549
550 if (defined $options) {
551 if (defined $options->{'do_wait'}) {
552 $do_wait = $options->{'do_wait'};
553 }
554 if (defined $options->{'output_verbosity'}) {
555 $output_verbosity = $options->{'output_verbosity'};
556 }
557 }
558
559 my $full_server_jar = $self->{'full_server_jar'};
560 my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
561
562 my $server_props = "-DSTOP.PORT=$jetty_stop_port";
563 $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
564 my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
565
566 print STDERR "**** java server stop cmd:\n $server_java_cmd\n" if ($output_verbosity>1);
567
568 if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
569
570 my $line;
571 while (defined($line=<STOPIN>)) {
572 print "Jetty shutdown: $line" if ($output_verbosity>1);
573 }
574 close(STOPIN);
575
576 if ($do_wait) {
577 wait(); # let the child process finish
578 }
579
580 if ($output_verbosity>0) {
581 print "Jetty server shutdown\n";
582 }
583 }
584 else {
585 print STDERR "Error: failed to stop solr-jetty-server\n";
586 print STDERR "$!\n";
587 print STDERR "Command attempted was:\n";
588 print STDERR " $server_java_cmd\n";
589 print STDERR "run from directory:\n";
590 print STDERR " $solr_home\n";
591 print STDERR "----\n";
592
593 exit -2;
594 }
595}
596
597
598
5991;
Note: See TracBrowser for help on using the repository browser.