root/gs3-extensions/solr/trunk/src/perllib/solrserver.pm @ 28182

Revision 28182, 14.0 KB (checked in by kjdon, 6 years ago)

passing the web/ext/solr dir as solr home to the server. So now it uses the solr.xml file there. We do all modifications in that file, no need to touch the ones in the original solr ext dir. Don't bother copying changes out to .in file each time, we will do that at the end of activate.pl. Once the collection is live, then we can save the changes to the .in file.

Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr/jetty server
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27package solrserver;
28
29use strict;
30#no strict 'refs';
31
32use solrutil;
33
34sub new {
35    my $class = shift(@_);
36    my ($build_dir) = @_;
37
38    my $self = { 'jetty_stop_key' => "greenstone-solr" };
39
40    $self->{'build_dir'} = $build_dir;
41
42    my $search_path = &solrutil::get_search_path();
43
44    my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
45    my $full_server_jar = solrutil::locate_file($search_path,$server_jar);
46    $self->{'full_server_jar'} = $full_server_jar;
47
48    $self->{'jetty_explicitly_started'} = undef;
49
50    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
51    my $base_url = "http://localhost:$jetty_server_port/solr/";
52    my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
53   
54    $self->{'base-url'} = $base_url;
55    $self->{'admin-url'} = $admin_url;
56
57    return bless $self, $class;
58}
59
60sub set_jetty_stop_key {
61    my $self = shift (@_);
62    my ($stop_key) = @_;
63
64    $self->{'jetty_stop_key'} = $stop_key if defined $stop_key;
65}
66
67sub _wget_service
68{
69    my $self = shift (@_);
70    my ($output_format,$url,$cgi_get_args) = @_;
71
72    my $full_url = $url;
73
74    $url .= "?$cgi_get_args" if (defined $cgi_get_args);
75   
76    my $cmd = "wget -O - \"$url\" 2>&1";
77
78    my $preamble_output = "";   
79    my $xml_output = "";
80    my $error_output = undef;
81
82    my $in_preamble = ($output_format eq "xml") ? 1 : 0;
83   
84##    print STDERR "**** wgetcmd = \n $cmd\n";
85
86    if (open(WIN,"$cmd |")) {
87
88    my $line;
89    while (defined ($line=<WIN>)) {
90
91        if ($line =~ m/ERROR \d+:/) {
92        chomp $line;
93        $error_output = $line;
94        last;
95        }
96        elsif ($line =~ m/failed: Connection refused/) {
97        chomp $line;
98        $error_output = $line;
99        last;
100        }
101        elsif ($in_preamble) {
102        if ($line =~ m/<.*>/) {
103            $in_preamble = 0;
104        }
105        else {
106            $preamble_output .= $line;
107        }
108        }
109
110        if (! $in_preamble) {
111        $xml_output .= $line;
112        }
113    }
114    close(WIN);
115
116    }
117    else {
118    $error_output = "Error: failed to run $cmd\n";
119    $error_output .= "  $!\n";
120    }
121
122    my $output = { 'url'      => $full_url,
123           'preamble' => $preamble_output,
124           'output'   => $xml_output,
125           'error'    => $error_output };
126
127    return $output;
128}
129
130
131sub _base_service
132{
133    my $self = shift (@_);
134    my ($cgi_get_args) = @_;
135
136    my $base_url = $self->{'base-url'};
137
138    return $self->_wget_service("html",$base_url,$cgi_get_args);
139}
140 
141sub _admin_service
142{
143    my $self = shift (@_);
144    my ($cgi_get_args) = @_;
145
146    my $admin_url = $self->{'admin-url'};
147
148    return $self->_wget_service("xml",$admin_url,$cgi_get_args);
149}
150
151
152sub server_running
153{
154    my $self = shift @_;
155
156    my $output = $self->_base_service();
157
158    my $have_error = defined $output->{'error'};
159
160    my $running = !$have_error;
161
162    return $running;
163}
164
165
166sub admin_ping_core
167{
168    my $self = shift @_;
169    my ($core) = @_;
170
171    my $cgi_get_args = "action=STATUS&core=$core";
172
173    my $ping_status = 1;
174
175    my $output = $self->_admin_service($cgi_get_args);
176
177    if (defined $output->{'error'}) {
178    # severe error, such as failing to connect to the server
179    $ping_status = 0;
180
181    my $url      = $output->{'url'};
182    my $preamble = $output->{'preamble'};
183    my $error    = $output->{'error'};
184   
185    print STDERR "----\n";
186    print STDERR "Error: Failed to get XML response from:\n";
187    print STDERR "         $url\n";
188    print STDERR "Output was:\n";
189    print STDERR $preamble if ($preamble ne "");
190    print STDERR "$error\n";
191    print STDERR "----\n";
192    }
193    else {
194   
195    # If the collection doesn't exist yet, then there will be
196    # an empty element of the form:
197    #   <lst name="collect-doc"/>
198    # where 'collect' is the actual name of the collection,
199    # such as demo
200
201    my $xml_output = $output->{'output'};
202   
203    my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
204   
205    $ping_status = !($xml_output =~ m/$empty_element/s);
206    }
207
208    return $ping_status;
209}
210
211sub filtered_copy
212{
213    my $self = shift @_;
214
215    my $src_file = shift @_;
216    my $dst_file = shift @_;
217    my $re_substitutions = shift @_;
218
219    # $re_substitutions is a hashmap of the form: [re_key] => subst_str
220   
221    my $content = "";
222
223    if (open(FIN,'<:utf8',$src_file)) {
224
225    my $line;
226    while (defined($line=<FIN>)) {
227        $content .= $line;
228    }
229    }
230
231    close(FIN);
232
233    # perform RE string substitutions
234    foreach my $re_key (keys %$re_substitutions) {
235
236    my $subst_str = $re_substitutions->{$re_key};
237
238    # Perform substitution of the form:
239    #  $content =~ s/$re_key/$subst_str/g;
240    # but allow allow separator char (default '/')
241    # and flags (default 'g') to be parameterized
242
243    $content =~ s/$re_key/$subst_str/g;
244    }
245   
246    if (open(FOUT, '>:utf8', $dst_file)) {
247    print FOUT $content;
248    close(FOUT);
249    }
250    else {
251    print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
252    }   
253}
254
255sub solr_xml_to_solr_xml_in
256{
257    my $self = shift @_;
258    my ($solr_xml_dir) = @_;
259   
260    my $gsdl3home = $ENV{'GSDL3HOME'};
261   
262    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
263    # if not passed in, use stored solr_live_home
264    $solr_xml_dir = $self->{'solr_live_home'};
265    }
266    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
267    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
268
269    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
270
271    my $replacement_map = { qr/$gsdl3home_re/ => "\@gsdl3home\@" };
272
273    $self->filtered_copy($solrxml,$solrxml_in,$replacement_map);
274}
275
276
277sub solr_xml_in_to_solr_xml
278{
279    my $self = shift @_;
280    my ($solr_xml_dir) = @_;
281
282    my $gsdl3home = $ENV{'GSDL3HOME'};
283    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
284    # if not passed in, use stored solr home
285    $solr_xml_dir = $self->{'solr_live_home'};
286    }
287    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
288    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
289   
290    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
291
292    my $replacement_map = { qr/\@gsdl3home\@/ => $gsdl3home_re };
293
294    $self->filtered_copy($solrxml_in,$solrxml,$replacement_map);
295}
296
297
298# Some of the Solr CoreAdmin API calls available.
299# See http://wiki.apache.org/solr/CoreAdmin
300sub admin_reload_core
301{
302    my $self = shift @_;
303    my ($core) = @_;
304
305    my $cgi_get_args = "action=RELOAD&core=$core";
306
307    $self->_admin_service($cgi_get_args);
308
309}
310
311sub admin_rename_core
312{
313    my $self = shift @_;
314    my ($oldcore, $newcore) = @_;
315
316    my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
317
318    $self->_admin_service($cgi_get_args);
319
320}
321
322sub admin_swap_core
323{
324    my $self = shift @_;
325    my ($oldcore, $newcore) = @_;
326
327    my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
328
329    $self->_admin_service($cgi_get_args);
330
331}
332
333# The ALIAS action is not supported in our version of solr (despite it
334# being marked as experimental in the documentation for Core Admin)
335sub admin_alias_core
336{
337    my $self = shift @_;
338    my ($oldcore, $newcore) = @_;
339
340    my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
341
342    $self->_admin_service($cgi_get_args);
343
344}
345
346sub admin_create_core
347{
348    my $self = shift @_;
349    my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
350
351    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
352
353    my $cgi_get_args = "action=CREATE&name=$core";
354
355    my $collect_home = $ENV{'GSDLCOLLECTDIR'};
356    my $etc_dirname = &util::filename_cat($collect_home,"etc");
357
358    if(!defined $data_parent_dir) {
359    $data_parent_dir = $self->{'build_dir'};
360    }
361   
362    my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir" 
363       
364    $cgi_get_args .= "&instanceDir=$etc_dirname";
365    $cgi_get_args .= "&dataDir=$idx_dirname";
366
367    $self->_admin_service($cgi_get_args);
368
369}
370
371# removes (unloads) core from the ext/solr/sorl.xml config file
372sub admin_unload_core
373{
374    my $self = shift @_;
375    my ($core, $delete) = @_;
376
377    my $cgi_get_args = "action=UNLOAD&core=$core"; # &deleteIndex=true from Solr3.3
378    if(defined $delete && $delete == 1) {
379    $cgi_get_args = $cgi_get_args."&deleteIndex=true";
380    }
381
382    $self->_admin_service($cgi_get_args);
383
384}
385
386sub start
387{
388    my $self = shift @_;
389    my ($verbosity) = @_;
390   
391    $verbosity = 1 unless defined $verbosity;
392
393    my $solr_home         = $ENV{'GEXT_SOLR'};
394    my $solr_live_home    = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr");
395    my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'};
396    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
397
398    chdir($solr_home);
399    #$self->{'solr_home'} = $solr_home;
400    $self->{'solr_live_home'} = $solr_live_home;
401##    my $solr_etc = &util::filename_cat($solr_home,"etc");
402
403    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
404    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
405    #$server_props .= " -Dsolr.solr.home=$solr_home";
406$server_props .= " -Dsolr.solr.home=$solr_live_home";
407    my $full_server_jar = $self->{'full_server_jar'};
408   
409    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
410
411
412    my $server_status = "unknown";
413
414    if ($self->server_running()) {
415    $server_status = "already-running";
416    }
417    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
418
419    print STDERR "**** starting up solr jetty server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
420
421    my $line;
422    while (defined($line=<STARTIN>)) {
423        # Scan through output until you see a line like:
424        #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
425        # which signifies that the server has started up and is
426        # "ready and listening"
427   
428        if (($line =~ m/^(WARN|ERROR|SEVERE):/)
429        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
430        print "Jetty startup: $line";
431        }
432       
433        if ($line =~ m/WARN::failed SocketConnector/) {
434        if ($line =~ m/Address already in use/) {
435            $server_status = "already-running";
436        }
437        else {
438            $server_status = "failed-to-start";
439        }
440        last;
441        }
442       
443        if ($line =~ m/INFO::Started SocketConnector/) {
444        $server_status = "explicitly-started";
445        last;
446        }
447    }
448    }
449    else {
450    print STDERR "Error: failed to start solr-jetty-server\n";
451    print STDERR "$!\n";
452    print STDERR "Command attempted was:\n";
453    print STDERR "  $server_java_cmd\n";
454    print STDERR "run from directory:\n";
455    print STDERR "  $solr_home\n";
456    print STDERR "----\n";
457
458    exit -1;
459    }
460
461    if ($server_status eq "explicitly-started") {
462    $self->{'jetty_explicitly_started'} = 1;
463    print "Jetty server ready and listening for connections on port";
464    print " $jetty_server_port\n";
465       
466    # now we know the server is ready to accept connections, fork a
467    # child process that continues to listen to the output and
468    # prints out any lines that are not INFO lines
469
470    if (fork()==0) {
471        # child process
472
473        my $line;
474        while (defined ($line = <STARTIN>)) {
475
476#       if (($line =~ m/^(WARN|ERROR|SEVERE):/)
477#           || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
478#           print "Jetty $line";
479#       }
480
481        # skip info lines
482        next if ($line =~ m/^INFO:/);
483        next if ($line =~ m/^[0-9 :-]*INFO::/);
484        next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
485        next if ($line =~ m/^\d{4}-\d{2}-\d{2}\s+/);
486
487##      next if ($line =~ m/^\s*\w+{.*}/);
488
489        # if here, then some non-trival message has been logged
490        print "Jetty/Solr processing: $line";
491        }
492        close(STARTIN);
493       
494        # And now stop nicely
495        exit 0;
496    }
497    # otherwise let the parent continue on
498    }
499    elsif ($server_status eq "already-running") {
500    print STDERR "Using existing server detected on port $jetty_server_port\n";
501    $self->{'jetty_explicitly_started'} = 0;
502    }
503    elsif ($server_status eq "failed-to-start") {
504    print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
505    print STDERR ", but encountered an initialization error\n";
506    exit -1;
507    }
508
509}
510
511sub explicitly_started
512{
513    my $self = shift @_;
514
515    return $self->{'jetty_explicitly_started'};
516}
517
518sub stop
519{   
520    my $self = shift @_;
521    my ($options) = @_;
522
523    my $solr_home         = $ENV{'GEXT_SOLR'};
524
525    chdir($solr_home);
526
527    # defaults
528    my $do_wait = 1;
529    my $output_verbosity = 1;
530
531    if (defined $options) {
532    if (defined $options->{'do_wait'}) {
533        $do_wait = $options->{'do_wait'};
534    }
535    if (defined $options->{'output_verbosity'}) {
536        $output_verbosity = $options->{'output_verbosity'};
537    }
538    }
539
540    my $full_server_jar = $self->{'full_server_jar'};
541    my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
542   
543    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
544    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
545    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
546
547    print STDERR "**** java server stop cmd:\n  $server_java_cmd\n" if ($output_verbosity>1);
548
549    if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
550
551    my $line;
552    while (defined($line=<STOPIN>)) {
553        print "Jetty shutdown: $line" if ($output_verbosity>1);
554    }
555    close(STOPIN);
556
557    if ($do_wait) {
558        wait(); # let the child process finish
559    }
560
561    if ($output_verbosity>0) {
562        print "Jetty server shutdown\n";
563    }
564    }
565    else {
566    print STDERR "Error: failed to stop solr-jetty-server\n";
567    print STDERR "$!\n";
568    print STDERR "Command attempted was:\n";
569    print STDERR "  $server_java_cmd\n";
570    print STDERR "run from directory:\n";
571    print STDERR "  $solr_home\n";
572    print STDERR "----\n";
573
574    exit -2;
575    }
576}
577
578
579
5801;
Note: See TracBrowser for help on using the browser.