root/gs3-extensions/solr/trunk/src/perllib/solrserver.pm @ 29349

Revision 29349, 15.3 KB (checked in by ak19, 5 years ago)

Fixing the problem on Mac Lion binaries (not svn) of wget not running when solrserver.pm invokes it, which results in rebuilding solr collections from GLI or from the cmd line to fail. If the gnome-lib env is set up, then the gs2build/bin/darwin/wget executable of Lion binaries works. Don't want to adjust the overall environment of the terminal, as that can mess up display of graphical programs like emacs at least on linux. Instead, calling util::set_gnomelib_env() that was introduced earlier this year before running the wget command. Doing so sets the gnome-lib env for the wget cmd. This is the solution that was since recently already being used in gs2build/bin/script/gsWget.pl

Line 
1###########################################################################
2#
3# solrserver.pm -- class for starting and stopping the Solr/jetty server
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26
27package solrserver;
28
29use strict;
30#no strict 'refs';
31
32use solrutil;
33
34sub new {
35    my $class = shift(@_);
36    my ($build_dir) = @_;
37
38    my $self = { 'jetty_stop_key' => "greenstone-solr" };
39
40    $self->{'build_dir'} = $build_dir;
41
42    my $search_path = &solrutil::get_search_path();
43
44    my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
45    my $full_server_jar = solrutil::locate_file($search_path,$server_jar);
46    $self->{'full_server_jar'} = $full_server_jar;
47
48    $self->{'jetty_explicitly_started'} = undef;
49
50    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
51    my $base_url = "http://localhost:$jetty_server_port/solr/";
52    my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
53   
54    $self->{'base-url'} = $base_url;
55    $self->{'admin-url'} = $admin_url;
56
57    return bless $self, $class;
58}
59
60sub set_jetty_stop_key {
61    my $self = shift (@_);
62    my ($stop_key) = @_;
63
64    $self->{'jetty_stop_key'} = $stop_key if defined $stop_key;
65}
66
67sub _wget_service
68{
69    my $self = shift (@_);
70    my ($output_format,$url,$cgi_get_args) = @_;
71
72    my $full_url = $url;
73
74    $url .= "?$cgi_get_args" if (defined $cgi_get_args);
75
76##    print STDERR "\n\n**** _wget_service SOLR WEB URL: $url\n\n";
77   
78    # the wget binary is dependent on the gnomelib_env (particularly lib/libiconv2.dylib) being set, particularly on Mac Lion binaries (android too?)
79    &util::set_gnomelib_env(); # this will set the gnomelib env once for each subshell launched, by first checking if GEXTGNOME is not already set
80
81    my $cmd = "wget -O - \"$url\" 2>&1";
82
83    my $preamble_output = "";   
84    my $xml_output = "";
85    my $error_output = undef;
86
87    my $in_preamble = ($output_format eq "xml") ? 1 : 0;
88   
89##    print STDERR "**** wgetcmd = \n $cmd\n";
90
91    if (open(WIN,"$cmd |")) {
92
93    my $line;
94    while (defined ($line=<WIN>)) {
95
96        if ($line =~ m/ERROR \d+:/) {
97        chomp $line;
98        $error_output = $line;
99        last;
100        }
101        elsif ($line =~ m/failed: Connection refused/) {
102        chomp $line;
103        $error_output = $line;
104        last;
105        }
106        elsif ($in_preamble) {
107        if ($line =~ m/<.*>/) {
108            $in_preamble = 0;
109        }
110        else {
111            $preamble_output .= $line;
112        }
113        }
114
115        if (! $in_preamble) {
116        $xml_output .= $line;
117        }
118    }
119    close(WIN);
120
121    }
122    else {
123    $error_output = "Error: failed to run $cmd\n";
124    $error_output .= "  $!\n";
125    }
126
127    if(defined $error_output) {
128    print STDERR "\n\n**** WGET_SERVICE got an error: $error_output\n\n";
129    }
130
131    my $output = { 'url'      => $full_url,
132           'preamble' => $preamble_output,
133           'output'   => $xml_output,
134           'error'    => $error_output };
135
136    return $output;
137}
138
139
140sub _base_service
141{
142    my $self = shift (@_);
143    my ($cgi_get_args) = @_;
144
145    my $base_url = $self->{'base-url'};
146
147    return $self->_wget_service("html",$base_url,$cgi_get_args);
148}
149 
150sub _admin_service
151{
152    my $self = shift (@_);
153    my ($cgi_get_args) = @_;
154
155    my $admin_url = $self->{'admin-url'};
156
157    return $self->_wget_service("xml",$admin_url,$cgi_get_args);
158}
159
160
161sub server_running
162{
163    my $self = shift @_;
164
165    my $output = $self->_base_service();
166
167    my $have_error = defined $output->{'error'};
168
169    my $running = ($have_error) ? 0 : 1;
170
171    return $running;
172}
173
174
175sub admin_ping_core
176{
177    my $self = shift @_;
178    my ($core) = @_;
179
180    my $cgi_get_args = "action=STATUS&core=$core";
181
182    my $ping_status = 1;
183
184    my $output = $self->_admin_service($cgi_get_args);
185
186    if (defined $output->{'error'}) {
187    # severe error, such as failing to connect to the server
188    $ping_status = 0;
189
190    my $url      = $output->{'url'};
191    my $preamble = $output->{'preamble'};
192    my $error    = $output->{'error'};
193   
194    print STDERR "----\n";
195    print STDERR "Error: Failed to get XML response from:\n";
196    print STDERR "         $url\n";
197    print STDERR "Output was:\n";
198    print STDERR $preamble if ($preamble ne "");
199    print STDERR "$error\n";
200    print STDERR "----\n";
201    }
202    else {
203   
204    # If the collection doesn't exist yet, then there will be
205    # an empty element of the form:
206    #   <lst name="collect-doc"/>
207    # where 'collect' is the actual name of the collection,
208    # such as demo
209
210    my $xml_output = $output->{'output'};
211   
212    my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
213   
214    $ping_status = !($xml_output =~ m/$empty_element/s);
215    }
216
217    return $ping_status;
218}
219
220sub filtered_copy
221{
222    my $self = shift @_;
223
224    my $src_file = shift @_;
225    my $dst_file = shift @_;
226    my $re_substitutions = shift @_;
227
228    # $re_substitutions is a hashmap of the form: [re_key] => subst_str
229   
230    my $content = "";
231
232    if (open(FIN,'<:utf8',$src_file)) {
233
234    my $line;
235    while (defined($line=<FIN>)) {
236        $content .= $line;
237    }
238    }
239
240    close(FIN);
241
242    # perform RE string substitutions
243    foreach my $re_key (keys %$re_substitutions) {
244
245    my $subst_str = $re_substitutions->{$re_key};
246
247    # Perform substitution of the form:
248    #  $content =~ s/$re_key/$subst_str/g;
249    # but allow allow separator char (default '/')
250    # and flags (default 'g') to be parameterized
251
252    $content =~ s/$re_key/$subst_str/g;
253    }
254   
255    if (open(FOUT, '>:utf8', $dst_file)) {
256    print FOUT $content;
257    close(FOUT);
258    }
259    else {
260    print STDERR "Error: Failed to open file '$dst_file' for writing.\n$!\n";
261    }   
262}
263
264sub solr_xml_to_solr_xml_in
265{
266    my $self = shift @_;
267    my ($solr_xml_dir) = @_;
268   
269    my $gsdl3home = $ENV{'GSDL3HOME'};
270   
271    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
272    # if not passed in, use stored solr_live_home
273    $solr_xml_dir = $self->{'solr_live_home'};
274    }
275    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
276    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
277
278    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
279
280    my $replacement_map = { qr/$gsdl3home_re/ => "\@gsdl3home\@" };
281
282    $self->filtered_copy($solrxml,$solrxml_in,$replacement_map);
283}
284
285
286sub solr_xml_in_to_solr_xml
287{
288    my $self = shift @_;
289    my ($solr_xml_dir) = @_;
290
291    my $gsdl3home = $ENV{'GSDL3HOME'};
292    if (!defined $solr_xml_dir || !-d $solr_xml_dir) {
293    # if not passed in, use stored solr home
294    $solr_xml_dir = $self->{'solr_live_home'};
295    }
296    my $solrxml_in = &util::filename_cat($solr_xml_dir, "solr.xml.in");
297    my $solrxml = &util::filename_cat($solr_xml_dir, "solr.xml");
298   
299    my $gsdl3home_re = &util::filename_to_regex($gsdl3home);
300
301    my $replacement_map = { qr/\@gsdl3home\@/ => $gsdl3home_re };
302
303    $self->filtered_copy($solrxml_in,$solrxml,$replacement_map);
304}
305
306
307# Some of the Solr CoreAdmin API calls available.
308# See http://wiki.apache.org/solr/CoreAdmin
309sub admin_reload_core
310{
311    my $self = shift @_;
312    my ($core) = @_;
313
314    my $cgi_get_args = "action=RELOAD&core=$core";
315
316    $self->_admin_service($cgi_get_args);
317
318}
319
320sub admin_rename_core
321{
322    my $self = shift @_;
323    my ($oldcore, $newcore) = @_;
324
325    my $cgi_get_args = "action=RENAME&core=$oldcore&other=$newcore";
326
327    $self->_admin_service($cgi_get_args);
328
329}
330
331sub admin_swap_core
332{
333    my $self = shift @_;
334    my ($oldcore, $newcore) = @_;
335
336    my $cgi_get_args = "action=SWAP&core=$oldcore&other=$newcore";
337
338    $self->_admin_service($cgi_get_args);
339
340}
341
342# The ALIAS action is not supported in our version of solr (despite it
343# being marked as experimental in the documentation for Core Admin)
344sub admin_alias_core
345{
346    my $self = shift @_;
347    my ($oldcore, $newcore) = @_;
348
349    my $cgi_get_args = "action=ALIAS&core=$oldcore&other=$newcore";
350
351    $self->_admin_service($cgi_get_args);
352
353}
354
355sub admin_create_core
356{
357    my $self = shift @_;
358    my ($core, $data_parent_dir) = @_; # data_parent_dir is optional, can be index_dir. Defaults to builddir if not provided
359
360    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
361
362    my $cgi_get_args = "action=CREATE&name=$core";
363
364    my $collect_home = $ENV{'GSDLCOLLECTDIR'};
365    my $etc_dirname = &util::filename_cat($collect_home,"etc");
366
367    if(!defined $data_parent_dir) {
368    $data_parent_dir = $self->{'build_dir'};
369    }
370   
371    my $idx_dirname = &util::filename_cat($data_parent_dir,$ds_idx); # "dataDir" 
372       
373    $cgi_get_args .= "&instanceDir=$etc_dirname";
374    $cgi_get_args .= "&dataDir=$idx_dirname";
375
376    $self->_admin_service($cgi_get_args);
377
378}
379
380# removes (unloads) core from the ext/solr/sorl.xml config file
381sub admin_unload_core
382{
383    my $self = shift @_;
384    my ($core, $delete) = @_;
385
386    my $cgi_get_args = "action=UNLOAD&core=$core"; # &deleteIndex=true from Solr3.3
387    if(defined $delete && $delete == 1) {
388    $cgi_get_args = $cgi_get_args."&deleteIndex=true";
389    }
390
391    $self->_admin_service($cgi_get_args);
392
393}
394
395sub start
396{
397    my $self = shift @_;
398    my ($verbosity) = @_;
399   
400    $verbosity = 1 unless defined $verbosity;
401
402    my $solr_home         = $ENV{'GEXT_SOLR'};
403    my $solr_live_home    = &util::filename_cat($ENV{'GSDL3HOME'}, "ext", "solr");
404    my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'};
405    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
406
407    chdir($solr_home);
408    #$self->{'solr_home'} = $solr_home;
409    $self->{'solr_live_home'} = $solr_live_home;
410##    my $solr_etc = &util::filename_cat($solr_home,"etc");
411
412    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
413    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
414    #$server_props .= " -Dsolr.solr.home=$solr_home";
415$server_props .= " -Dsolr.solr.home=$solr_live_home";
416    my $full_server_jar = $self->{'full_server_jar'};
417   
418    # https://wiki.apache.org/solr/SolrLogging
419    my $solr_slf4j    = &util::filename_cat($solr_home, "lib", "ext", "*");
420    my $solr_lib_java    = &util::filename_cat($solr_home, "lib", "java", "*");
421    my $solr_jar_dep = &util::pathname_cat($solr_lib_java, $solr_slf4j);
422   
423    my $solr_log4j    = &util::filename_cat($solr_home, "conf", "log4j.properties");   
424    if (defined $ENV{'GSDLOS'} && (($ENV{'GSDLOS'} =~ m/windows/) && ($^O ne "cygwin"))) {
425        $solr_log4j =~ s@\\@\/@g;
426        $solr_log4j = "/$solr_log4j"; # Three slashes on windows: file///c:/path/to/log4j.properties
427    }
428   
429    #my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_home/lib/java/*:$solr_slf4j/*\" $server_props -jar \"$full_server_jar\"";
430    my $server_java_cmd = "java -Dlog4j.configuration=file://$solr_log4j -classpath \"$solr_jar_dep\" $server_props -jar \"$full_server_jar\"";
431    my $server_status = "unknown";
432
433    if ($self->server_running()) {
434    $server_status = "already-running";
435    }
436    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
437
438    print STDERR "**** starting up solr jetty server with cmd start =\n $server_java_cmd\n" if ($verbosity > 1);
439
440    my $line;
441    while (defined($line=<STARTIN>)) {
442        # Scan through output until you see a line like:
443        #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
444        # which signifies that the server has started up and is
445        # "ready and listening"
446   
447        if (($line =~ m/^(WARN|ERROR|SEVERE):/)
448        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
449        print "Jetty startup: $line";
450        }
451       
452        if ($line =~ m/WARN.*failed SocketConnector/) {
453        if ($line =~ m/Address already in use/) {
454            $server_status = "already-running";
455        }
456        else {
457            $server_status = "failed-to-start";
458        }
459        last;
460        }
461       
462        if ($line =~ m/INFO.*Started SocketConnector/) {
463        $server_status = "explicitly-started";
464        last;
465        }
466    }
467    }
468    else {
469    print STDERR "Error: failed to start solr-jetty-server\n";
470    print STDERR "$!\n";
471    print STDERR "Command attempted was:\n";
472    print STDERR "  $server_java_cmd\n";
473    print STDERR "run from directory:\n";
474    print STDERR "  $solr_home\n";
475    print STDERR "----\n";
476
477    exit -1;
478    }
479
480    if ($server_status eq "explicitly-started") {
481    $self->{'jetty_explicitly_started'} = 1;
482    print "Jetty server ready and listening for connections on port";
483    print " $jetty_server_port\n";
484       
485    # now we know the server is ready to accept connections, fork a
486    # child process that continues to listen to the output and
487    # prints out any lines that are not INFO lines
488
489    if (fork()==0) {
490        # child process
491
492        my $line;
493        while (defined ($line = <STARTIN>)) {
494
495#       if (($line =~ m/^(WARN|ERROR|SEVERE):/)
496#           || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
497#           print "Jetty $line";
498#       }
499
500        # skip info lines
501        next if ($line =~ m/^INFO:/);
502        next if ($line =~ m/^[0-9 :-]*INFO::/);
503        next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
504        next if ($line =~ m/^\d{4}-\d{2}-\d{2}\s+/);
505
506##      next if ($line =~ m/^\s*\w+{.*}/);
507
508        # if here, then some non-trival message has been logged
509        print "Jetty/Solr processing: $line";
510        }
511        close(STARTIN);
512       
513        # And now stop nicely
514        exit 0;
515    }
516    # otherwise let the parent continue on
517    }
518    elsif ($server_status eq "already-running") {
519    print STDERR "Using existing server detected on port $jetty_server_port\n";
520    $self->{'jetty_explicitly_started'} = 0;
521    }
522    elsif ($server_status eq "failed-to-start") {
523    print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
524    print STDERR ", but encountered an initialization error\n";
525    exit -1;
526    }
527
528}
529
530sub explicitly_started
531{
532    my $self = shift @_;
533
534    return $self->{'jetty_explicitly_started'};
535}
536
537sub stop
538{   
539    my $self = shift @_;
540    my ($options) = @_;
541
542    my $solr_home         = $ENV{'GEXT_SOLR'};
543
544    chdir($solr_home);
545
546    # defaults
547    my $do_wait = 1;
548    my $output_verbosity = 1;
549
550    if (defined $options) {
551    if (defined $options->{'do_wait'}) {
552        $do_wait = $options->{'do_wait'};
553    }
554    if (defined $options->{'output_verbosity'}) {
555        $output_verbosity = $options->{'output_verbosity'};
556    }
557    }
558
559    my $full_server_jar = $self->{'full_server_jar'};
560    my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
561   
562    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
563    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
564    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
565
566    print STDERR "**** java server stop cmd:\n  $server_java_cmd\n" if ($output_verbosity>1);
567
568    if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
569
570    my $line;
571    while (defined($line=<STOPIN>)) {
572        print "Jetty shutdown: $line" if ($output_verbosity>1);
573    }
574    close(STOPIN);
575
576    if ($do_wait) {
577        wait(); # let the child process finish
578    }
579
580    if ($output_verbosity>0) {
581        print "Jetty server shutdown\n";
582    }
583    }
584    else {
585    print STDERR "Error: failed to stop solr-jetty-server\n";
586    print STDERR "$!\n";
587    print STDERR "Command attempted was:\n";
588    print STDERR "  $server_java_cmd\n";
589    print STDERR "run from directory:\n";
590    print STDERR "  $solr_home\n";
591    print STDERR "----\n";
592
593    exit -2;
594    }
595}
596
597
598
5991;
Note: See TracBrowser for help on using the browser.