source: gs3-extensions/solr/trunk/src/perllib/solrserver.pm@ 27858

Last change on this file since 27858 was 27858, checked in by davidb, 8 years ago

Methods added to generate solr.xml.in from solr.xml and vice versa. Routines to help with installing Greenstone from a binary release, and having the solr.xml file just right (i.e. with the correct value of GSDL3HOME for the users install) so the solr-jdbm-demo collection.

File size: 14.3 KB
Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr/jetty server
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27package solrserver;
28
29use strict;
30#no strict 'refs';
31
32use solrutil;
33
34sub new {
35 my $class = shift(@_);
36 my ($build_dir) = @_;
37
38 my $self = { 'jetty_stop_key' => "greenstone-solr" };
39
40 $self->{'build_dir'} = $build_dir;
41
42 my $search_path = &solrutil::get_search_path();
43
44 my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
45 my $full_server_jar = solrutil::locate_file($search_path,$server_jar);
46 $self->{'full_server_jar'} = $full_server_jar;
47
48 $self->{'jetty_explicitly_started'} = undef;
49
50 my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
51 my $base_url = "http://localhost:$jetty_server_port/solr/";
52 my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
53
54 $self->{'base-url'} = $base_url;
55 $self->{'admin-url'} = $admin_url;
56
57 return bless $self, $class;
58}
59
60sub set_jetty_stop_key {
61 my $self = shift (@_);
62 my ($stop_key) = @_;
63
64 $self->{'jetty_stop_key'} = $stop_key if defined $stop_key;
65}
66
67sub _wget_service
68{
69 my $self = shift (@_);
70 my ($output_format,$url,$cgi_get_args) = @_;
71
72 my $full_url = $url;
73
74 $url .= "?$cgi_get_args" if (defined $cgi_get_args);
75
76 my $cmd = "wget -O - \"$url\" 2>&1";
77
78 my $preamble_output = "";
79 my $xml_output = "";
80 my $error_output = undef;
81
82 my $in_preamble = ($output_format eq "xml") ? 1 : 0;
83
84## print STDERR "**** wgetcmd = \n $cmd\n";
85
86 if (open(WIN,"$cmd |")) {
87
88 my $line;
89 while (defined ($line=<WIN>)) {
90
91 if ($line =~ m/ERROR \d+:/) {
92 chomp $line;
93 $error_output = $line;
94 last;
95 }
96 elsif ($line =~ m/failed: Connection refused/) {
97 chomp $line;
98 $error_output = $line;
99 last;
100 }
101 elsif ($in_preamble) {
102 if ($line =~ m/<.*>/) {
103 $in_preamble = 0;
104 }
105 else {
106 $preamble_output .= $line;
107 }
108 }
109
110 if (! $in_preamble) {
111 $xml_output .= $line;
112 }
113 }
114 close(WIN);
115
116 }
117 else {
118 $error_output = "Error: failed to run $cmd\n";
119 $error_output .= " $!\n";
120 }
121
122 my $output = { 'url' => $full_url,
123 'preamble' => $preamble_output,
124 'output' => $xml_output,
125 'error' => $error_output };
126
127 return $output;
128}
129
130
131sub _base_service
132{
133 my $self = shift (@_);
134 my ($cgi_get_args) = @_;
135
136 my $base_url = $self->{'base-url'};
137
138 return $self->_wget_service("html",$base_url,$cgi_get_args);
139}
140
141sub _admin_service
142{
143 my $self = shift (@_);
144 my ($cgi_get_args) = @_;
145
146 my $admin_url = $self->{'admin-url'};
147
148 return $self->_wget_service("xml",$admin_url,$cgi_get_args);
149}
150
151
152sub server_running
153{
154 my $self = shift @_;
155
156 my $output = $self->_base_service();
157
158 my $have_error = defined $output->{'error'};
159
160 my $running = !$have_error;
161
162 return $running;
163}
164
165
166sub admin_ping_core
167{
168 my $self = shift @_;
169 my ($core) = @_;
170
171 my $cgi_get_args = "action=STATUS&core=$core";
172
173 my $ping_status = 1;
174
175 my $output = $self->_admin_service($cgi_get_args);
176
177 if (defined $output->{'error'}) {
178 # severe error, such as failing to connect to the server
179 $ping_status = 0;
180
181 my $url = $output->{'url'};
182 my $preamble = $output->{'preamble'};
183 my $error = $output->{'error'};
184
185 print STDERR "----\n";
186 print STDERR "Error: Failed to get XML response from:\n";
187 print STDERR " $url\n";
188 print STDERR "Output was:\n";
189 print STDERR $preamble if ($preamble ne "");
190 print STDERR "$error\n";
191 print STDERR "----\n";
192 }
193 else {
194
195 # If the collection doesn't exist yet, then there will be
196 # an empty element of the form:
197 # <lst name="collect-doc"/>
198 # where 'collect' is the actual name of the collection,
199 # such as demo
200
201 my $xml_output = $output->{'output'};
202
203 my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
204
205 $ping_status = !($xml_output =~ m/$empty_element/s);
206 }
207
208 return $ping_status;
209}
210
211sub filtered_copy
212{
213 my $self = shift @_;
214 my ($src_file,$dst_file,$re_substitutions) = @_;
215
216 my $sep = shift @_ || "/";
217 my $flags = shift @_ || "g";
218
219 # $re_substitutions is a hashmap of the form: [re_key] => subst_str
220
221 my $content = "";
222
223 if (open(FIN,'<:utf8',$src_file)) {
224
225 my $line;
226 while (defined($line=<FIN>)) {
227 $content .= $line;
228 }
229 }
230
231 close(FIN);
232
233 # perform RE string substitutions
234 foreach my $re_key (keys %$re_substitutions) {
235
236 my $subst_str = $re_substitutions->{$re_key};
237
238 # Perform substitution of the form:
239 # $content =~ s/$re_key/$subst_str/g;
240 # but allow allow separator char (default '/')
241 # and flags (default 'g') to be parameterized
242
243 my $eval_str = "\$content =~ s$sep$re_key$sep$subst_str$sep$flags";
244
245 eval {
246 $eval_str;
247 };
248 if ($@) {
249 print STDERR "Warning: failed to evaluate\n $eval_str\n$@\n";
250 }
251
252 }
253
254 if (open(FOUT, '>:utf8', $dst_file)) {
255 print FOUT $content;
256 close(FOUT);
257 }
258 else {
259 print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
260 }
261}
262
263sub solr_xml_to_solr_xml_in
264{
265 my $self = shift @_;
266
267 my $gsdl3home = $ENV{'GSDL3HOME'};
268 my $web_solr_ext_dir = &util::filename_cat($gsdl3home, "ext", "solr");
269 my $web_solrxml_in = &util::filename_cat($web_solr_ext_dir, "solr.xml.in");
270 my $web_solrxml = &util::filename_cat($web_solr_ext_dir, "solr.xml");
271
272 my $replacement_map = { "$gsdl3home" => "\\\@gsdl3home\\\@" };
273
274 $self->filtered_copy($web_solrxml,$web_solrxml_in,$replacement_map, "^", "g");
275}
276
277
278sub solr_xml_in_to_solr_xml
279{
280 my $self = shift @_;
281
282 my $gsdl3home = $ENV{'GSDL3HOME'};
283 my $web_solr_ext_dir = &util::filename_cat($gsdl3home, "ext", "solr");
284 my $web_solrxml_in = &util::filename_cat($web_solr_ext_dir, "solr.xml.in");
285 my $web_solrxml = &util::filename_cat($web_solr_ext_dir, "solr.xml");
286
287 my $replacement_map = { "\\\@gsdl3home\\\@" => "$gsdl3home" };
288
289 $self->filtered_copy($web_solrxml_in,$web_solrxml,$replacement_map, "^", "g");
290}
291
292
293# Some of the Solr CoreAdmin API calls available.
294# See http://wiki.apache.org/solr/CoreAdmin
295sub admin_reload_core
296{
297 my $self = shift @_;
298 my ($core) = @_;
299
300 my $cgi_get_args = "action=RELOAD&core=$core";
301
302 $self->_admin_service($cgi_get_args);
303
304 $self->solr_xml_to_solr_xml_in();
305}
306
307sub admin_rename_core
308{
309 my $self = shift @_;
310 my ($oldcore, $newcore) = @_;
311
312 my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
313
314 $self->_admin_service($cgi_get_args);
315
316 $self->solr_xml_to_solr_xml_in();
317}
318
319sub admin_swap_core
320{
321 my $self = shift @_;
322 my ($oldcore, $newcore) = @_;
323
324 my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
325
326 $self->_admin_service($cgi_get_args);
327
328 $self->solr_xml_to_solr_xml_in();
329}
330
331# The ALIAS action is not supported in our version of solr (despite it
332# being marked as experimental in the documentation for Core Admin)
333sub admin_alias_core
334{
335 my $self = shift @_;
336 my ($oldcore, $newcore) = @_;
337
338 my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
339
340 $self->_admin_service($cgi_get_args);
341
342 $self->solr_xml_to_solr_xml_in();
343}
344
345sub admin_create_core
346{
347 my $self = shift @_;
348 my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
349
350 my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
351
352 my $cgi_get_args = "action=CREATE&name=$core";
353
354 my $collect_home = $ENV{'GSDLCOLLECTDIR'};
355 my $etc_dirname = &util::filename_cat($collect_home,"etc");
356
357 if(!defined $data_parent_dir) {
358 $data_parent_dir = $self->{'build_dir'};
359 }
360
361 my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir"
362
363 $cgi_get_args .= "&instanceDir=$etc_dirname";
364 $cgi_get_args .= "&dataDir=$idx_dirname";
365
366 $self->_admin_service($cgi_get_args);
367
368 $self->solr_xml_to_solr_xml_in();
369}
370
371# removes (unloads) core from the ext/solr/sorl.xml config file
372sub admin_unload_core
373{
374 my $self = shift @_;
375 my ($core, $delete) = @_;
376
377 my $cgi_get_args = "action=UNLOAD&core=$core"; # &deleteIndex=true from Solr3.3
378 if(defined $delete && $delete == 1) {
379 $cgi_get_args = $cgi_get_args."&deleteIndex=true";
380 }
381
382 $self->_admin_service($cgi_get_args);
383
384 $self->solr_xml_to_solr_xml_in();
385}
386
387sub copy_solrxml_to_web
388{
389 my $self = shift @_;
390
391 my $ext_solrxml = &util::filename_cat($ENV{'GEXT_SOLR'}, "solr.xml.in");
392 my $web_solrxml = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr", "solr.xml.in");
393
394 #print STDERR "@@@@ Copying $ext_solrxml to $web_solrxml...\n";
395
396 &FileUtils::copyFiles($ext_solrxml, $web_solrxml);
397
398 $self->solr_xml_in_to_solr_xml();
399}
400
401sub start
402{
403 my $self = shift @_;
404 my ($verbosity) = @_;
405
406 $verbosity = 1 unless defined $verbosity;
407
408 my $solr_home = $ENV{'GEXT_SOLR'};
409 my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
410 my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
411
412 chdir($solr_home);
413
414## my $solr_etc = &util::filename_cat($solr_home,"etc");
415
416 my $server_props = "-DSTOP.PORT=$jetty_stop_port";
417 $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
418 $server_props .= " -Dsolr.solr.home=$solr_home";
419
420 my $full_server_jar = $self->{'full_server_jar'};
421
422 my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
423
424
425 my $server_status = "unknown";
426
427 if ($self->server_running()) {
428 $server_status = "already-running";
429 }
430 elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
431
432 print STDERR "**** starting up solr jetty server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
433
434 my $line;
435 while (defined($line=<STARTIN>)) {
436 # Scan through output until you see a line like:
437 # 2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
438 # which signifies that the server has started up and is
439 # "ready and listening"
440
441 if (($line =~ m/^(WARN|ERROR|SEVERE):/)
442 || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
443 print "Jetty startup: $line";
444 }
445
446 if ($line =~ m/WARN::failed SocketConnector/) {
447 if ($line =~ m/Address already in use/) {
448 $server_status = "already-running";
449 }
450 else {
451 $server_status = "failed-to-start";
452 }
453 last;
454 }
455
456 if ($line =~ m/INFO::Started SocketConnector/) {
457 $server_status = "explicitly-started";
458 last;
459 }
460 }
461 }
462 else {
463 print STDERR "Error: failed to start solr-jetty-server\n";
464 print STDERR "$!\n";
465 print STDERR "Command attempted was:\n";
466 print STDERR " $server_java_cmd\n";
467 print STDERR "run from directory:\n";
468 print STDERR " $solr_home\n";
469 print STDERR "----\n";
470
471 exit -1;
472 }
473
474 if ($server_status eq "explicitly-started") {
475 $self->{'jetty_explicitly_started'} = 1;
476 print "Jetty server ready and listening for connections on port";
477 print " $jetty_server_port\n";
478
479 # now we know the server is ready to accept connections, fork a
480 # child process that continues to listen to the output and
481 # prints out any lines that are not INFO lines
482
483 if (fork()==0) {
484 # child process
485
486 my $line;
487 while (defined ($line = <STARTIN>)) {
488
489# if (($line =~ m/^(WARN|ERROR|SEVERE):/)
490# || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
491# print "Jetty $line";
492# }
493
494 # skip info lines
495 next if ($line =~ m/^INFO:/);
496 next if ($line =~ m/^[0-9 :-]*INFO::/);
497 next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
498 next if ($line =~ m/^\d{4}-\d{2}-\d{2}\s+/);
499
500## next if ($line =~ m/^\s*\w+{.*}/);
501
502 # if here, then some non-trival message has been logged
503 print "Jetty/Solr processing: $line";
504 }
505 close(STARTIN);
506
507 # And now stop nicely
508 exit 0;
509 }
510 # otherwise let the parent continue on
511 }
512 elsif ($server_status eq "already-running") {
513 print STDERR "Using existing server detected on port $jetty_server_port\n";
514 $self->{'jetty_explicitly_started'} = 0;
515 }
516 elsif ($server_status eq "failed-to-start") {
517 print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
518 print STDERR ", but encountered an initialization error\n";
519 exit -1;
520 }
521
522}
523
524sub explicitly_started
525{
526 my $self = shift @_;
527
528 return $self->{'jetty_explicitly_started'};
529}
530
531sub stop
532{
533 my $self = shift @_;
534 my ($options) = @_;
535
536 my $solr_home = $ENV{'GEXT_SOLR'};
537
538 chdir($solr_home);
539
540 # defaults
541 my $do_wait = 1;
542 my $output_verbosity = 1;
543
544 if (defined $options) {
545 if (defined $options->{'do_wait'}) {
546 $do_wait = $options->{'do_wait'};
547 }
548 if (defined $options->{'output_verbosity'}) {
549 $output_verbosity = $options->{'output_verbosity'};
550 }
551 }
552
553 my $full_server_jar = $self->{'full_server_jar'};
554 my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
555
556 my $server_props = "-DSTOP.PORT=$jetty_stop_port";
557 $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
558 my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
559
560 print STDERR "**** java server stop cmd:\n $server_java_cmd\n" if ($output_verbosity>1);
561
562 if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
563
564 my $line;
565 while (defined($line=<STOPIN>)) {
566 print "Jetty shutdown: $line" if ($output_verbosity>1);
567 }
568 close(STOPIN);
569
570 if ($do_wait) {
571 wait(); # let the child process finish
572 }
573
574 if ($output_verbosity>0) {
575 print "Jetty server shutdown\n";
576 }
577 }
578 else {
579 print STDERR "Error: failed to stop solr-jetty-server\n";
580 print STDERR "$!\n";
581 print STDERR "Command attempted was:\n";
582 print STDERR " $server_java_cmd\n";
583 print STDERR "run from directory:\n";
584 print STDERR " $solr_home\n";
585 print STDERR "----\n";
586
587 exit -2;
588 }
589}
590
591
592
5931;
Note: See TracBrowser for help on using the repository browser.