source: gs3-extensions/solr/trunk/src/perllib/solrserver.pm@ 31490

Last change on this file since 31490 was 31490, checked in by ak19, 7 years ago
  1. Fix to issue of a tomcat host/port change not propagating to solr host/port change when rebuilding a solr collection after tomcat host/port change. The change to tomcat server props need to be made after gs3-setup.sh was already run in the terminal earlier, to encouner the problem upon solr build. The bug was reproduced on Linux, and the fix for it also tested on Linux. Still need to test fix out on Windows. 2. Simultaneously made http protocol used in solr more robust to whether it's http or https.
File size: 14.0 KB
Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr with the
4# GS3 tomcat server.
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright (C) 1999 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26
27
28package solrserver;
29
30use strict;
31#no strict 'refs';
32
33use solrutil;
34
35sub new {
36 my $class = shift(@_);
37 my ($build_dir) = @_;
38
39 my $self = { 'build_dir' => $build_dir };
40
41 my $search_path = &solrutil::get_search_path();
42
43 $self->{'server_explicitly_started'} = undef;
44
45 # set SOLR_HOST and SOLR_PORT env vars (tomcat host and port, if not using jetty)
46 # by calling ant get-default-solr-servlet if possible. Else fallback on whatever the existing env vars are.
47 # tomcat host and port would have been set up in the env as SOLR_HOST and SOLR_PORT
48 # In case someone changed the tomcat host/port, we want to update the solr server variables too
49 my $solr_url = &solrutil::get_solr_servlet_url();
50 # get the url parts, though we won't be using most of them
51 my ($protocol, $server_host, $server_port, $servlet_name) = &solrutil::get_solr_url_parts($solr_url);
52
53 # set the solr server env vars to what was discovered, so that any other old perl code
54 # dependent on these env vars will have any changes propagated.
55 # (All perl code referencing these env vars should already be updated, but still...)
56 $ENV{'SOLR_HOST'} = $server_host;
57 $ENV{'SOLR_PORT'} = $server_port;
58
59 $self->{'base-url'} = $solr_url; # e.g. of the form http://localhost:8383/solr
60 $self->{'admin-url'} = "$solr_url/admin/cores";
61
62 return bless $self, $class;
63}
64
65sub get_solr_base_url {
66 my $self = shift (@_);
67 return $self->{'base-url'};
68}
69
70sub _wget_service
71{
72 my $self = shift (@_);
73 my ($output_format,$url,$cgi_get_args) = @_;
74
75 my $full_url = $url;
76
77 $url .= "?$cgi_get_args" if (defined $cgi_get_args);
78
79 print STDERR "\n\n**** _wget_service SOLR WEB URL: $url\n\n";
80
81 # the wget binary is dependent on the gnomelib_env (particularly lib/libiconv2.dylib) being set, particularly on Mac Lion binaries (android too?)
82 &util::set_gnomelib_env(); # this will set the gnomelib env once for each subshell launched, by first checking if GEXTGNOME is not already set
83
84 my $cmd = "wget -O - \"$url\" 2>&1";
85
86 my $preamble_output = "";
87 my $xml_output = "";
88 my $error_output = undef;
89
90 my $in_preamble = ($output_format eq "xml") ? 1 : 0;
91
92## print STDERR "**** wgetcmd = \n $cmd\n";
93
94 if (open(WIN,"$cmd |")) {
95
96 my $line;
97 while (defined ($line=<WIN>)) {
98
99 if ($line =~ m/ERROR \d+:/) {
100 chomp $line;
101 $error_output = $line;
102 last;
103 }
104 elsif ($line =~ m/failed: Connection refused/) {
105 chomp $line;
106 $error_output = $line;
107 last;
108 }
109 elsif ($in_preamble) {
110 if ($line =~ m/<.*>/) {
111 $in_preamble = 0;
112 }
113 else {
114 $preamble_output .= $line;
115 }
116 }
117
118 if (! $in_preamble) {
119 $xml_output .= $line;
120 }
121 }
122 close(WIN);
123
124 }
125 else {
126 $error_output = "Error: failed to run $cmd\n";
127 $error_output .= " $!\n";
128 }
129
130 if(defined $error_output) {
131 print STDERR "\n\n**** WGET_SERVICE got an error: $error_output\n\n";
132 }
133
134 my $output = { 'url' => $full_url,
135 'preamble' => $preamble_output,
136 'output' => $xml_output,
137 'error' => $error_output };
138
139 return $output;
140}
141
142
143sub _base_service
144{
145 my $self = shift (@_);
146 my ($cgi_get_args) = @_;
147
148 my $base_url = $self->{'base-url'};
149
150 return $self->_wget_service("html",$base_url,$cgi_get_args);
151}
152
153sub _admin_service
154{
155 my $self = shift (@_);
156 my ($cgi_get_args) = @_;
157
158 my $admin_url = $self->{'admin-url'};
159
160 return $self->_wget_service("xml",$admin_url,$cgi_get_args);
161}
162
163
164sub server_running
165{
166 my $self = shift @_;
167
168 my $output = $self->_base_service();
169
170 my $have_error = defined $output->{'error'};
171
172 my $running = ($have_error) ? 0 : 1;
173
174 return $running;
175}
176
177
178sub admin_ping_core
179{
180 my $self = shift @_;
181 my ($core) = @_;
182
183 my $cgi_get_args = "action=STATUS&core=$core";
184
185 my $ping_status = 1;
186
187 my $output = $self->_admin_service($cgi_get_args);
188
189 if (defined $output->{'error'}) {
190 # severe error, such as failing to connect to the server
191 $ping_status = 0;
192
193 my $url = $output->{'url'};
194 my $preamble = $output->{'preamble'};
195 my $error = $output->{'error'};
196
197 print STDERR "----\n";
198 print STDERR "Error: Failed to get XML response from:\n";
199 print STDERR " $url\n";
200 print STDERR "Output was:\n";
201 print STDERR $preamble if ($preamble ne "");
202 print STDERR "$error\n";
203 print STDERR "----\n";
204 }
205 else {
206
207 # If the collection doesn't exist yet, then there will be
208 # an empty element of the form:
209 # <lst name="collect-doc"/>
210 # where 'collect' is the actual name of the collection,
211 # such as demo
212
213 my $xml_output = $output->{'output'};
214
215 my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
216
217 $ping_status = !($xml_output =~ m/$empty_element/s);
218 }
219
220 return $ping_status;
221}
222
223sub filtered_copy
224{
225 my $self = shift @_;
226
227 my $src_file = shift @_;
228 my $dst_file = shift @_;
229 my $re_substitutions = shift @_;
230
231 # $re_substitutions is a hashmap of the form: [re_key] => subst_str
232
233 my $content = "";
234
235 if (open(FIN,'<:utf8',$src_file)) {
236
237 my $line;
238 while (defined($line=<FIN>)) {
239 $content .= $line;
240 }
241 }
242
243 close(FIN);
244
245 # perform RE string substitutions
246 foreach my $re_key (keys %$re_substitutions) {
247
248 my $subst_str = $re_substitutions->{$re_key};
249
250 # Perform substitution of the form:
251 # $content =~ s/$re_key/$subst_str/g;
252 # but allow allow separator char (default '/')
253 # and flags (default 'g') to be parameterized
254
255 $content =~ s/$re_key/$subst_str/g;
256 }
257
258 if (open(FOUT, '>:utf8', $dst_file)) {
259 print FOUT $content;
260 close(FOUT);
261 }
262 else {
263 print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
264 }
265}
266
267sub solr_xml_to_solr_xml_in
268{
269 my $self = shift @_;
270 my ($solr_xml_dir) = @_;
271
272 my $gsdl3home = $ENV{'GSDL3HOME'};
273
274 if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
275 # if not passed in, use stored solr_live_home
276 $solr_xml_dir = $self->{'solr_live_home'};
277 }
278
279 my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
280 my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
281
282 my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
283
284 my $replacement_map = { qr/$gsdl3home_re/ => "\@gsdl3home\@" };
285
286 $self->filtered_copy($solrxml,$solrxml_in,$replacement_map);
287}
288
289
290sub solr_xml_in_to_solr_xml
291{
292 my $self = shift @_;
293 my ($solr_xml_dir) = @_;
294
295 my $gsdl3home = $ENV{'GSDL3HOME'};
296 if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
297 # if not passed in, use stored solr home
298 $solr_xml_dir = $self->{'solr_live_home'};
299 }
300 my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
301 my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
302
303 my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
304
305 my $replacement_map = { qr/\@gsdl3home\@/ => $gsdl3home_re };
306
307 $self->filtered_copy($solrxml_in,$solrxml,$replacement_map);
308}
309
310
311# Some of the Solr CoreAdmin API calls available.
312# See http://wiki.apache.org/solr/CoreAdmin
313sub admin_reload_core
314{
315 my $self = shift @_;
316 my ($core) = @_;
317
318 my $cgi_get_args = "action=RELOAD&core=$core";
319
320 $self->_admin_service($cgi_get_args);
321
322}
323
324sub admin_rename_core
325{
326 my $self = shift @_;
327 my ($oldcore, $newcore) = @_;
328
329 my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
330
331 $self->_admin_service($cgi_get_args);
332
333}
334
335sub admin_swap_core
336{
337 my $self = shift @_;
338 my ($oldcore, $newcore) = @_;
339
340 my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
341
342 $self->_admin_service($cgi_get_args);
343
344}
345
346# The ALIAS action is not supported in our version of solr (despite it
347# being marked as experimental in the documentation for Core Admin)
348sub admin_alias_core
349{
350 my $self = shift @_;
351 my ($oldcore, $newcore) = @_;
352
353 my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
354
355 $self->_admin_service($cgi_get_args);
356
357}
358
359sub admin_create_core
360{
361 my $self = shift @_;
362 my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
363
364 my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
365
366 my $cgi_get_args = "action=CREATE&name=$core";
367
368 my $collect_home = $ENV{'GSDLCOLLECTDIR'};
369 my $etc_dirname = &util::filename_cat($collect_home,"etc");
370
371 if(!defined $data_parent_dir) {
372 $data_parent_dir = $self->{'build_dir'};
373 }
374
375 my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir"
376
377 $cgi_get_args .= "&instanceDir=$etc_dirname";
378 $cgi_get_args .= "&dataDir=$idx_dirname";
379
380 $self->_admin_service($cgi_get_args);
381
382}
383
384# removes (unloads) core from the ext/solr/sorl.xml config file
385sub admin_unload_core
386{
387 my $self = shift @_;
388 my ($core, $delete) = @_;
389
390 my $cgi_get_args = "action=UNLOAD&core=$core";
391 # &deleteIndex=true available from Solr3.3, see https://wiki.apache.org/solr/CoreAdmin.
392 # Also available since later Solr versions: deleteDataDir and deleteInstanceDir
393 if(defined $delete && $delete == 1) {
394 $cgi_get_args = $cgi_get_args."&deleteIndex=true";
395 }
396
397 $self->_admin_service($cgi_get_args);
398
399}
400
401sub start
402{
403 my $self = shift @_;
404 my ($verbosity) = @_;
405
406 $verbosity = 1 unless defined $verbosity;
407
408 my $solr_live_home = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr");
409 $self->{'solr_live_home'} = $solr_live_home; # will be used later to generate solr.xml.in from solr.xml and vice-versa
410 my $server_port = $ENV{'SOLR_PORT'};
411 my $server_host = $ENV{'SOLR_HOST'};
412
413 chdir($ENV{'GSDL3SRCHOME'});
414
415 my $server_java_cmd = "ant start";
416
417 my $server_status = "unknown";
418
419 if ($self->server_running()) {
420 $server_status = "already-running";
421 print STDERR "@@@@ server already running\n\n";
422 }
423 elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
424
425 print STDERR "@@@@ need to start tomcat\n\n";
426 print STDERR "**** starting up tomcat server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
427
428 my $line;
429 while (defined($line=<STARTIN>)) {
430
431 #if ($line =~ m/^(BUILD FAILED)/) {
432 print "Tomcat startup: $line";
433 #}
434 if ($line =~ m/^BUILD SUCCESSFUL/) {
435 last;
436 }
437 }
438
439 close(STARTIN);
440
441 if ($self->server_running()) {
442 $server_status = "explicitly-started";
443 #print STDERR "\n*** Tomcat server has started up now.\n\n";
444 } else {
445 $server_status = "failed-to-start"; # no need to set this, will be exiting below anyway
446
447 print STDERR "Error: failed to start greenstone tomcat server\n";
448 print STDERR "$!\n";
449 print STDERR "Command attempted was:\n";
450 print STDERR " $server_java_cmd\n";
451 print STDERR "run from directory:\n";
452 print STDERR " $ENV{'GSDL3SRCHOME'}\n";
453 print STDERR "----\n";
454
455 exit -1;
456 }
457 }
458 else {
459 print STDERR "@@@@ failed to start tomcat\n\n";
460 $server_status = "failed-to-start"; # no need to set this, will be exiting below anyway
461
462 print STDERR "Error: unable to start greenstone tomcat server\n";
463 print STDERR "$!\n";
464 print STDERR "Command attempted was:\n";
465 print STDERR " $server_java_cmd\n";
466 print STDERR "run from directory:\n";
467 print STDERR " $ENV{'GSDL3SRCHOME'}\n";
468 print STDERR "----\n";
469
470 exit -1;
471 }
472
473 if ($server_status eq "explicitly-started") {
474 $self->{'server_explicitly_started'} = 1;
475 print "Tomcat server ready and listening for connections at ";
476 print " $server_host:$server_port\n";
477
478 # now we know the server is ready to accept connections
479 }
480 elsif ($server_status eq "already-running") {
481 print STDERR "Using existing tomcat server detected at $server_host:$server_port\n";
482 $self->{'server_explicitly_started'} = 0;
483 }
484 elsif ($server_status eq "failed-to-start") {
485 print STDERR "Started Solr/Tomcat web server at $server_host:$server_port";
486 print STDERR ", but encountered an initialization error\n";
487 exit -1;
488 }
489
490}
491
492sub explicitly_started
493{
494 my $self = shift @_;
495
496 return $self->{'server_explicitly_started'};
497}
498
499sub stop
500{
501 my $self = shift @_;
502 my ($options) = @_;
503
504 my $solr_home = $ENV{'GEXT_SOLR'};
505
506 chdir($ENV{'GSDL3SRCHOME'});
507
508 # defaults
509 my $do_wait = 1;
510 my $output_verbosity = 1;
511
512 if (defined $options) {
513 if (defined $options->{'do_wait'}) {
514 $do_wait = $options->{'do_wait'};
515 }
516 if (defined $options->{'output_verbosity'}) {
517 $output_verbosity = $options->{'output_verbosity'};
518 }
519 }
520
521 my $server_java_cmd = "ant stop";
522
523 print STDERR "**** java server stop cmd:\n $server_java_cmd\n" if ($output_verbosity>1);
524
525 if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
526
527 my $line;
528 while (defined($line=<STOPIN>)) {
529 print "@@@@ Tomcat shutdown: $line"; #if ($output_verbosity>1);
530 }
531 close(STOPIN);
532
533 if ($do_wait) {
534 wait(); # let the child process finish
535 }
536
537 if ($output_verbosity>0) {
538 print "@@@@@ Tomcat server shutdown\n";
539 }
540 }
541 else {
542 print STDERR "Error: failed to stop tomcat-server\n";
543 print STDERR "$!\n";
544 print STDERR "Command attempted was:\n";
545 print STDERR " $server_java_cmd\n";
546 print STDERR "run from directory:\n";
547 print STDERR " $solr_home\n";
548 print STDERR "----\n";
549
550 exit -2;
551 }
552}
553
554
555
5561;
Note: See TracBrowser for help on using the repository browser.