root/gs3-extensions/solr/trunk/src/perllib/solrserver.pm @ 29219

Revision 29219, 15.0 KB (checked in by ak19, 5 years ago)

Debug statements printed to STDERR if there's any error when accessing the jetty solr urls

Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr/jetty server
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27package solrserver;
28
29use strict;
30#no strict 'refs';
31
32use solrutil;
33
34sub new {
35    my $class = shift(@_);
36    my ($build_dir) = @_;
37
38    my $self = { 'jetty_stop_key' => "greenstone-solr" };
39
40    $self->{'build_dir'} = $build_dir;
41
42    my $search_path = &solrutil::get_search_path();
43
44    my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
45    my $full_server_jar = solrutil::locate_file($search_path,$server_jar);
46    $self->{'full_server_jar'} = $full_server_jar;
47
48    $self->{'jetty_explicitly_started'} = undef;
49
50    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
51    my $base_url = "http://localhost:$jetty_server_port/solr/";
52    my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
53   
54    $self->{'base-url'} = $base_url;
55    $self->{'admin-url'} = $admin_url;
56
57    return bless $self, $class;
58}
59
60sub set_jetty_stop_key {
61    my $self = shift (@_);
62    my ($stop_key) = @_;
63
64    $self->{'jetty_stop_key'} = $stop_key if defined $stop_key;
65}
66
67sub _wget_service
68{
69    my $self = shift (@_);
70    my ($output_format,$url,$cgi_get_args) = @_;
71
72    my $full_url = $url;
73
74    $url .= "?$cgi_get_args" if (defined $cgi_get_args);
75
76##    print STDERR "\n\n**** _wget_service SOLR WEB URL: $url\n\n";
77   
78    my $cmd = "wget -O - \"$url\" 2>&1";
79
80    my $preamble_output = "";   
81    my $xml_output = "";
82    my $error_output = undef;
83
84    my $in_preamble = ($output_format eq "xml") ? 1 : 0;
85   
86##    print STDERR "**** wgetcmd = \n $cmd\n";
87
88    if (open(WIN,"$cmd |")) {
89
90    my $line;
91    while (defined ($line=<WIN>)) {
92
93        if ($line =~ m/ERROR \d+:/) {
94        chomp $line;
95        $error_output = $line;
96        last;
97        }
98        elsif ($line =~ m/failed: Connection refused/) {
99        chomp $line;
100        $error_output = $line;
101        last;
102        }
103        elsif ($in_preamble) {
104        if ($line =~ m/<.*>/) {
105            $in_preamble = 0;
106        }
107        else {
108            $preamble_output .= $line;
109        }
110        }
111
112        if (! $in_preamble) {
113        $xml_output .= $line;
114        }
115    }
116    close(WIN);
117
118    }
119    else {
120    $error_output = "Error: failed to run $cmd\n";
121    $error_output .= "  $!\n";
122    }
123
124    if(defined $error_output) {
125    print STDERR "\n\n**** WGET_SERVICE got an error: $error_output\n\n";
126    }
127
128    my $output = { 'url'      => $full_url,
129           'preamble' => $preamble_output,
130           'output'   => $xml_output,
131           'error'    => $error_output };
132
133    return $output;
134}
135
136
137sub _base_service
138{
139    my $self = shift (@_);
140    my ($cgi_get_args) = @_;
141
142    my $base_url = $self->{'base-url'};
143
144    return $self->_wget_service("html",$base_url,$cgi_get_args);
145}
146 
147sub _admin_service
148{
149    my $self = shift (@_);
150    my ($cgi_get_args) = @_;
151
152    my $admin_url = $self->{'admin-url'};
153
154    return $self->_wget_service("xml",$admin_url,$cgi_get_args);
155}
156
157
158sub server_running
159{
160    my $self = shift @_;
161
162    my $output = $self->_base_service();
163
164    my $have_error = defined $output->{'error'};
165
166    my $running = ($have_error) ? 0 : 1;
167
168    return $running;
169}
170
171
172sub admin_ping_core
173{
174    my $self = shift @_;
175    my ($core) = @_;
176
177    my $cgi_get_args = "action=STATUS&core=$core";
178
179    my $ping_status = 1;
180
181    my $output = $self->_admin_service($cgi_get_args);
182
183    if (defined $output->{'error'}) {
184    # severe error, such as failing to connect to the server
185    $ping_status = 0;
186
187    my $url      = $output->{'url'};
188    my $preamble = $output->{'preamble'};
189    my $error    = $output->{'error'};
190   
191    print STDERR "----\n";
192    print STDERR "Error: Failed to get XML response from:\n";
193    print STDERR "         $url\n";
194    print STDERR "Output was:\n";
195    print STDERR $preamble if ($preamble ne "");
196    print STDERR "$error\n";
197    print STDERR "----\n";
198    }
199    else {
200   
201    # If the collection doesn't exist yet, then there will be
202    # an empty element of the form:
203    #   <lst name="collect-doc"/>
204    # where 'collect' is the actual name of the collection,
205    # such as demo
206
207    my $xml_output = $output->{'output'};
208   
209    my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
210   
211    $ping_status = !($xml_output =~ m/$empty_element/s);
212    }
213
214    return $ping_status;
215}
216
217sub filtered_copy
218{
219    my $self = shift @_;
220
221    my $src_file = shift @_;
222    my $dst_file = shift @_;
223    my $re_substitutions = shift @_;
224
225    # $re_substitutions is a hashmap of the form: [re_key] => subst_str
226   
227    my $content = "";
228
229    if (open(FIN,'<:utf8',$src_file)) {
230
231    my $line;
232    while (defined($line=<FIN>)) {
233        $content .= $line;
234    }
235    }
236
237    close(FIN);
238
239    # perform RE string substitutions
240    foreach my $re_key (keys %$re_substitutions) {
241
242    my $subst_str = $re_substitutions->{$re_key};
243
244    # Perform substitution of the form:
245    #  $content =~ s/$re_key/$subst_str/g;
246    # but allow allow separator char (default '/')
247    # and flags (default 'g') to be parameterized
248
249    $content =~ s/$re_key/$subst_str/g;
250    }
251   
252    if (open(FOUT, '>:utf8', $dst_file)) {
253    print FOUT $content;
254    close(FOUT);
255    }
256    else {
257    print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
258    }   
259}
260
261sub solr_xml_to_solr_xml_in
262{
263    my $self = shift @_;
264    my ($solr_xml_dir) = @_;
265   
266    my $gsdl3home = $ENV{'GSDL3HOME'};
267   
268    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
269    # if not passed in, use stored solr_live_home
270    $solr_xml_dir = $self->{'solr_live_home'};
271    }
272    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
273    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
274
275    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
276
277    my $replacement_map = { qr/$gsdl3home_re/ => "\@gsdl3home\@" };
278
279    $self->filtered_copy($solrxml,$solrxml_in,$replacement_map);
280}
281
282
283sub solr_xml_in_to_solr_xml
284{
285    my $self = shift @_;
286    my ($solr_xml_dir) = @_;
287
288    my $gsdl3home = $ENV{'GSDL3HOME'};
289    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
290    # if not passed in, use stored solr home
291    $solr_xml_dir = $self->{'solr_live_home'};
292    }
293    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
294    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
295   
296    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
297
298    my $replacement_map = { qr/\@gsdl3home\@/ => $gsdl3home_re };
299
300    $self->filtered_copy($solrxml_in,$solrxml,$replacement_map);
301}
302
303
304# Some of the Solr CoreAdmin API calls available.
305# See http://wiki.apache.org/solr/CoreAdmin
306sub admin_reload_core
307{
308    my $self = shift @_;
309    my ($core) = @_;
310
311    my $cgi_get_args = "action=RELOAD&core=$core";
312
313    $self->_admin_service($cgi_get_args);
314
315}
316
317sub admin_rename_core
318{
319    my $self = shift @_;
320    my ($oldcore, $newcore) = @_;
321
322    my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
323
324    $self->_admin_service($cgi_get_args);
325
326}
327
328sub admin_swap_core
329{
330    my $self = shift @_;
331    my ($oldcore, $newcore) = @_;
332
333    my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
334
335    $self->_admin_service($cgi_get_args);
336
337}
338
339# The ALIAS action is not supported in our version of solr (despite it
340# being marked as experimental in the documentation for Core Admin)
341sub admin_alias_core
342{
343    my $self = shift @_;
344    my ($oldcore, $newcore) = @_;
345
346    my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
347
348    $self->_admin_service($cgi_get_args);
349
350}
351
352sub admin_create_core
353{
354    my $self = shift @_;
355    my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
356
357    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
358
359    my $cgi_get_args = "action=CREATE&name=$core";
360
361    my $collect_home = $ENV{'GSDLCOLLECTDIR'};
362    my $etc_dirname = &util::filename_cat($collect_home,"etc");
363
364    if(!defined $data_parent_dir) {
365    $data_parent_dir = $self->{'build_dir'};
366    }
367   
368    my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir" 
369       
370    $cgi_get_args .= "&instanceDir=$etc_dirname";
371    $cgi_get_args .= "&dataDir=$idx_dirname";
372
373    $self->_admin_service($cgi_get_args);
374
375}
376
377# removes (unloads) core from the ext/solr/sorl.xml config file
378sub admin_unload_core
379{
380    my $self = shift @_;
381    my ($core, $delete) = @_;
382
383    my $cgi_get_args = "action=UNLOAD&core=$core"; # &deleteIndex=true from Solr3.3
384    if(defined $delete && $delete == 1) {
385    $cgi_get_args = $cgi_get_args."&deleteIndex=true";
386    }
387
388    $self->_admin_service($cgi_get_args);
389
390}
391
392sub start
393{
394    my $self = shift @_;
395    my ($verbosity) = @_;
396   
397    $verbosity = 1 unless defined $verbosity;
398
399    my $solr_home         = $ENV{'GEXT_SOLR'};
400    my $solr_live_home    = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr");
401    my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'};
402    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
403
404    chdir($solr_home);
405    #$self->{'solr_home'} = $solr_home;
406    $self->{'solr_live_home'} = $solr_live_home;
407##    my $solr_etc = &util::filename_cat($solr_home,"etc");
408
409    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
410    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
411    #$server_props .= " -Dsolr.solr.home=$solr_home";
412$server_props .= " -Dsolr.solr.home=$solr_live_home";
413    my $full_server_jar = $self->{'full_server_jar'};
414   
415    # https://wiki.apache.org/solr/SolrLogging
416    my $solr_slf4j    = &util::filename_cat($solr_home, "lib", "ext", "*");
417    my $solr_lib_java    = &util::filename_cat($solr_home, "lib", "java", "*");
418    my $solr_jar_dep = &util::pathname_cat($solr_lib_java, $solr_slf4j);
419   
420    my $solr_log4j    = &util::filename_cat($solr_home, "conf", "log4j.properties");   
421    if (defined $ENV{'GSDLOS'} && (($ENV{'GSDLOS'} =~ m/windows/) && ($^O ne "cygwin"))) {
422        $solr_log4j =~ s@\\@\/@g;
423        $solr_log4j = "/$solr_log4j"; # Three slashes on windows: file///c:/path/to/log4j.properties
424    }
425   
426    #my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_home/lib/java/*:$solr_slf4j/*\" $server_props -jar \"$full_server_jar\"";
427    my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_jar_dep\" $server_props -jar \"$full_server_jar\"";
428    my $server_status = "unknown";
429
430    if ($self->server_running()) {
431    $server_status = "already-running";
432    }
433    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
434
435    print STDERR "**** starting up solr jetty server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
436
437    my $line;
438    while (defined($line=<STARTIN>)) {
439        # Scan through output until you see a line like:
440        #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
441        # which signifies that the server has started up and is
442        # "ready and listening"
443   
444        if (($line =~ m/^(WARN|ERROR|SEVERE):/)
445        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
446        print "Jetty startup: $line";
447        }
448       
449        if ($line =~ m/WARN.*failed SocketConnector/) {
450        if ($line =~ m/Address already in use/) {
451            $server_status = "already-running";
452        }
453        else {
454            $server_status = "failed-to-start";
455        }
456        last;
457        }
458       
459        if ($line =~ m/INFO.*Started SocketConnector/) {
460        $server_status = "explicitly-started";
461        last;
462        }
463    }
464    }
465    else {
466    print STDERR "Error: failed to start solr-jetty-server\n";
467    print STDERR "$!\n";
468    print STDERR "Command attempted was:\n";
469    print STDERR "  $server_java_cmd\n";
470    print STDERR "run from directory:\n";
471    print STDERR "  $solr_home\n";
472    print STDERR "----\n";
473
474    exit -1;
475    }
476
477    if ($server_status eq "explicitly-started") {
478    $self->{'jetty_explicitly_started'} = 1;
479    print "Jetty server ready and listening for connections on port";
480    print " $jetty_server_port\n";
481       
482    # now we know the server is ready to accept connections, fork a
483    # child process that continues to listen to the output and
484    # prints out any lines that are not INFO lines
485
486    if (fork()==0) {
487        # child process
488
489        my $line;
490        while (defined ($line = <STARTIN>)) {
491
492#       if (($line =~ m/^(WARN|ERROR|SEVERE):/)
493#           || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
494#           print "Jetty $line";
495#       }
496
497        # skip info lines
498        next if ($line =~ m/^INFO:/);
499        next if ($line =~ m/^[0-9 :-]*INFO::/);
500        next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
501        next if ($line =~ m/^\d{4}-\d{2}-\d{2}\s+/);
502
503##      next if ($line =~ m/^\s*\w+{.*}/);
504
505        # if here, then some non-trival message has been logged
506        print "Jetty/Solr processing: $line";
507        }
508        close(STARTIN);
509       
510        # And now stop nicely
511        exit 0;
512    }
513    # otherwise let the parent continue on
514    }
515    elsif ($server_status eq "already-running") {
516    print STDERR "Using existing server detected on port $jetty_server_port\n";
517    $self->{'jetty_explicitly_started'} = 0;
518    }
519    elsif ($server_status eq "failed-to-start") {
520    print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
521    print STDERR ", but encountered an initialization error\n";
522    exit -1;
523    }
524
525}
526
527sub explicitly_started
528{
529    my $self = shift @_;
530
531    return $self->{'jetty_explicitly_started'};
532}
533
534sub stop
535{   
536    my $self = shift @_;
537    my ($options) = @_;
538
539    my $solr_home         = $ENV{'GEXT_SOLR'};
540
541    chdir($solr_home);
542
543    # defaults
544    my $do_wait = 1;
545    my $output_verbosity = 1;
546
547    if (defined $options) {
548    if (defined $options->{'do_wait'}) {
549        $do_wait = $options->{'do_wait'};
550    }
551    if (defined $options->{'output_verbosity'}) {
552        $output_verbosity = $options->{'output_verbosity'};
553    }
554    }
555
556    my $full_server_jar = $self->{'full_server_jar'};
557    my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
558   
559    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
560    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
561    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
562
563    print STDERR "**** java server stop cmd:\n  $server_java_cmd\n" if ($output_verbosity>1);
564
565    if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
566
567    my $line;
568    while (defined($line=<STOPIN>)) {
569        print "Jetty shutdown: $line" if ($output_verbosity>1);
570    }
571    close(STOPIN);
572
573    if ($do_wait) {
574        wait(); # let the child process finish
575    }
576
577    if ($output_verbosity>0) {
578        print "Jetty server shutdown\n";
579    }
580    }
581    else {
582    print STDERR "Error: failed to stop solr-jetty-server\n";
583    print STDERR "$!\n";
584    print STDERR "Command attempted was:\n";
585    print STDERR "  $server_java_cmd\n";
586    print STDERR "run from directory:\n";
587    print STDERR "  $solr_home\n";
588    print STDERR "----\n";
589
590    exit -2;
591    }
592}
593
594
595
5961;
Note: See TracBrowser for help on using the browser.