Changeset 24483 for gs3-extensions/solr


Ignore:
Timestamp:
2011-08-26T00:22:29+12:00 (12 years ago)
Author:
davidb
Message:
Reworking of code that detects existing running instance of Solr/Jetty server. This was due to Windows version of Perl not implementing '' on an open call. Code currently messy and needs a further tidy up.
Location:
gs3-extensions/solr/trunk/src
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24453 r24483  
    5858sub open_java_solr
    5959{
    60   my ($collect, $doc_tag_level,$full_builddir,$indexdir,$removeold) = @_;
     60  my ($collect, $ds_idx,$full_builddir,$indexdir,$removeold) = @_;
    6161
    6262  # if removeold set, then delete the curring $full_builddir
    … …  
    7474
    7575  # Now start up the solr-post command
    76   &solrutil::open_post_pipe($collect,$doc_tag_level);
     76  &solrutil::open_post_pipe($collect,$ds_idx);
    7777}
    7878
    … …  
    224224
    225225  if ($filtered_argc < 5) {
    226     print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" doc-tag-level build-dir index-name\n";
     226    print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" {d|s}idx build-dir index-name\n";
    227227    exit 1;
    228228  }
    … …  
    230230  my $collect       = $filtered_argv[0];
    231231  my $mode          = $filtered_argv[1];
    232   my $doc_tag_level = $filtered_argv[2];
     232  my $ds_idx        = $filtered_argv[2];
    233233  my $full_builddir = $filtered_argv[3];
    234234  my $indexdir      = $filtered_argv[4];
    … …  
    237237  # documents, not if we are just storing the text
    238238  if ($mode eq "index") {
    239     open_java_solr($collect, $doc_tag_level, $full_builddir, $indexdir, $removeold);
     239    open_java_solr($collect, $ds_idx, $full_builddir, $indexdir, $removeold);
    240240  }
    241241

  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24456 r24483  
    117117        print STDERR "Sections:      $solr_passes_sections\n";
    118118        print STDERR "Build Dir:     $build_dir\n";
    119         print STDERR "Cmd:           $solr_passes_exe $collection text $solr_passes_sections \"$build_dir\" \"dummy\"   $osextra\n";
    120     if (!open($handle, "| $solr_passes_exe $collection text $solr_passes_sections \"$build_dir\" \"dummy\"   $osextra"))
     119        print STDERR "Cmd:           $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra\n";
     120    if (!open($handle, "| $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra"))
    121121    {
    122122        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
    … …  
    272272
    273273
    274 sub solr_core_admin
    275 {
    276     my $self = shift (@_);
    277     my ($url) = @_;
    278 
    279     my $cmd = "wget -O - \"$url\" 2>&1";
    280 
    281     my $preamble_output = "";   
    282     my $xml_output = "";
    283     my $error_output = undef;
    284 
    285     my $in_preamble = 1;
    286    
    287     if (open(WIN,"$cmd |")) {
    288 
    289     my $line;
    290     while (defined ($line=<WIN>)) {
    291 
    292         if ($line =~ m/ERROR \d+:/) {
    293         chomp $line;
    294         $error_output = $line;
    295         last;
    296         }
    297         elsif ($in_preamble) {
    298         if ($line =~ m/<.*>/) {
    299             $in_preamble = 0;
    300         }
    301         else {
    302             $preamble_output .= $line;
    303         }
    304         }
    305 
    306         if (!$in_preamble) {
    307         $xml_output .= $line;
    308         }
    309     }
    310     close(WIN);
    311 
    312     }
    313     else {
    314     $error_output = "Error: failed to run $cmd\n";
    315     $error_output .= "  $!\n";
    316     }
    317 
    318     my $output = { 'preamble' => $preamble_output,
    319            'output'   => $xml_output,
    320            'error'    => $error_output };
    321 
    322     return $output;
    323 }
    324 
    325274sub pre_build_indexes
    326275{
    … …  
    415364    # => at most two cores <colname>-Doc and <colname>-Sec
    416365
    417     my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
    418     my $base_url = "http://localhost:$jetty_server_port/solr/admin/cores";
    419 
    420366    my $collection = $self->{'collection'};
    421367
    … …  
    427373    my ($pindex) = $level =~ /^(.)/;
    428374   
    429     my $llevel = $mgppbuilder::level_map{$level};
    430     my $core = $collection."-".lc($llevel);
    431    
    432    
    433     # if collect==core not already in solr.xml (check with STATUS)
    434     # => use CREATE API to add to solr.xml
     375##  my $llevel = $mgppbuilder::level_map{$level};
     376##  my $core = $collection."-".lc($llevel);
     377       
     378    my $core = $collection."-".$pindex.$idx;
     379
     380    # if collect==core already in solr.xml (check with STATUS)
     381    # => use RELOAD call to refresh fields now expressed in schema.xml
    435382    #
    436383    # else
    437     # => use RELOAD call to refresh fields now expressed in schema.xml
    438    
    439    
    440     my $check_core_url    = "$base_url?action=STATUS&core=$core";
    441     my $output = $self->solr_core_admin($check_core_url);
    442 
    443     if (defined $output->{'error'}) {
    444 
    445         my $preamble = $output->{'preamble'};
    446         my $error    = $output->{'error'};
    447 
    448         print STDERR "----\n";
    449         print STDERR "Error: Failed to get XML response from:\n";
    450         print STDERR "         $check_core_url\n";
    451         print STDERR "Output was:\n";
    452         print STDERR $preamble if ($preamble ne "");
    453         print STDERR "$error\n";
    454         print STDERR "----\n";
    455 
    456         next;
    457     }
    458    
    459     # If the collection doesn't exist yet, then there will be
    460     # an empty element of the form:
    461     #   <lst name="collect-doc"/>
    462     # where 'collect' is the actual name of the collection,
    463     # such as demo
    464 
    465     my $xml_output = $output->{'output'};
    466    
    467     my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
    468    
    469     my $check_core_exists = !($xml_output =~ m/$empty_element/s);
    470    
    471     if ($check_core_exists) {
    472        
    473         my $reload_core_url    = "$base_url?action=RELOAD&core=$core";
    474        
     384    # => use CREATE API to add to solr.xml
     385       
     386    my $check_core_exists = $solr_server->admin_ping_core($core);
     387       
     388    if ($check_core_exists) {       
    475389        print $outhandle "Reloading Solr core: $core\n";
    476         $self->solr_core_admin($reload_core_url);
     390        $solr_server->admin_reload_core($core);
    477391    }
    478392    else {
    479        
    480         my $collect_home = $ENV{'GSDLCOLLECTDIR'};
    481         my $etc_dirname = &util::filename_cat($collect_home,"etc");
    482        
    483         my $build_dir = $self->{'build_dir'};
    484         my $idx_dirname = &util::filename_cat($build_dir,$pindex.$idx);
    485        
    486         my $create_core_url    = "$base_url?action=CREATE&name=$core";
    487         $create_core_url .= "&instanceDir=$etc_dirname";
    488         $create_core_url .= "&dataDir=$idx_dirname";
    489        
    490393        print $outhandle "Creating Solr core: $core\n";
    491         $self->solr_core_admin($create_core_url);
     394        $solr_server->admin_create_core($core);
    492395    }
    493396    }
    … …  
    574477    } else {
    575478    my $collection = $self->{'collection'};
    576 
    577     print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $solr_passes_sections \"$build_dir\" \"$indexdir\"   $osextra\n";
    578     if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $solr_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
     479    my $ds_idx = $self->{'index_mapping'}->{$index};
     480
     481    print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra\n";
     482    if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra")) {
    579483        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
    580484        die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n";

  • gs3-extensions/solr/trunk/src/perllib/solrserver.pm

    r24453 r24483  
    2828
    2929use strict;
     30#no strict 'refs';
    3031
    3132use solrutil;
     33
     34my $key_count = 0;
    3235
    3336sub new {
    3437    my $class = shift(@_);
    3538
    36     my $self = { 'jetty_stop_key' => "greenstone-solr" };
     39    $key_count++;
     40    my $self = { 'jetty_stop_key' => "greenstone-solr-".$$."-".$key_count };
    3741
    3842    my $search_path = &solrutil::get_search_path();
    … …  
    4448    $self->{'jetty_explicitly_started'} = undef;
    4549
     50    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
     51    my $base_url = "http://localhost:$jetty_server_port/solr/";
     52    my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores";
     53   
     54    $self->{'base-url'} = $base_url;
     55    $self->{'admin-url'} = $admin_url;
     56
    4657    return bless $self, $class;
    4758}
     59
     60
     61
     62sub _wget_service
     63{
     64    my $self = shift (@_);
     65    my ($output_format,$url,$cgi_get_args) = @_;
     66
     67    my $full_url = $url;
     68
     69    $url .= "?$cgi_get_args" if (defined $cgi_get_args);
     70   
     71    my $cmd = "wget -O - \"$url\" 2>&1";
     72
     73##    print STDERR "**** wget cmd: $cmd\n";
     74
     75    my $preamble_output = "";   
     76    my $xml_output = "";
     77    my $error_output = undef;
     78
     79    my $in_preamble = ($output_format eq "xml") ? 1 : 0;
     80   
     81    if (open(WIN,"$cmd |")) {
     82
     83    my $line;
     84    while (defined ($line=<WIN>)) {
     85
     86        if ($line =~ m/ERROR \d+:/) {
     87        chomp $line;
     88        $error_output = $line;
     89        last;
     90        }
     91        elsif ($line =~ m/failed: Connection refused/) {
     92        chomp $line;
     93        $error_output = $line;
     94        last;
     95        }
     96        elsif ($in_preamble) {
     97        if ($line =~ m/<.*>/) {
     98            $in_preamble = 0;
     99        }
     100        else {
     101            $preamble_output .= $line;
     102        }
     103        }
     104
     105        if (! $in_preamble) {
     106        $xml_output .= $line;
     107        }
     108    }
     109    close(WIN);
     110
     111    }
     112    else {
     113    $error_output = "Error: failed to run $cmd\n";
     114    $error_output .= "  $!\n";
     115    }
     116
     117    my $output = { 'preamble' => $preamble_output,
     118           'output'   => $xml_output,
     119           'error'    => $error_output };
     120
     121    return $output;
     122}
     123
     124
     125sub _base_service
     126{
     127    my $self = shift (@_);
     128    my ($cgi_get_args) = @_;
     129
     130    my $base_url = $self->{'base-url'};
     131
     132    return $self->_wget_service("html",$base_url,$cgi_get_args);
     133}
     134 
     135sub _admin_service
     136{
     137    my $self = shift (@_);
     138    my ($cgi_get_args) = @_;
     139
     140    my $admin_url = $self->{'admin-url'};
     141
     142    return $self->_wget_service("xml",$admin_url,$cgi_get_args);
     143}
     144
     145
     146sub server_running
     147{
     148    my $self = shift @_;
     149
     150    my $output = $self->_base_service();
     151
     152    my $have_error = defined $output->{'error'};
     153
     154    my $running = !$have_error;
     155
     156    return $running;
     157}
     158
     159
     160sub admin_ping_core
     161{
     162    my $self = shift @_;
     163    my ($core) = @_;
     164
     165    my $cgi_get_args = "action=STATUS&core=$core";
     166
     167    my $ping_status = 1;
     168
     169    my $output = $self->_admin_service($cgi_get_args);
     170
     171    if (defined $output->{'error'}) {
     172    # severe error, such as failing to connect to the server
     173    $ping_status = 0;
     174
     175    my $url      = $output->{'url'};
     176    my $preamble = $output->{'preamble'};
     177    my $error    = $output->{'error'};
     178   
     179    print STDERR "----\n";
     180    print STDERR "Error: Failed to get XML response from:\n";
     181    print STDERR "         $url\n";
     182    print STDERR "Output was:\n";
     183    print STDERR $preamble if ($preamble ne "");
     184    print STDERR "$error\n";
     185    print STDERR "----\n";
     186    }
     187    else {
     188   
     189    # If the collection doesn't exist yet, then there will be
     190    # an empty element of the form:
     191    #   <lst name="collect-doc"/>
     192    # where 'collect' is the actual name of the collection,
     193    # such as demo
     194
     195    my $xml_output = $output->{'output'};
     196   
     197    my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>";
     198   
     199    $ping_status = !($xml_output =~ m/$empty_element/s);
     200    }
     201
     202
     203    return $ping_status;
     204}
     205
     206
     207sub admin_reload_core
     208{
     209    my $self = shift @_;
     210    my ($core) = @_;
     211
     212    my $cgi_get_args = "action=RELOAD&core=$core";
     213
     214    $self->_admin_service($cgi_get_args);
     215}
     216
     217
     218sub admin_create_core
     219{
     220    my $self = shift @_;
     221    my ($core) = @_;
     222
     223    my ($ds_idx) = ($core =~ m/^.*-(.*)$/);
     224
     225    my $cgi_get_args = "action=CREATE&name=$core";
     226
     227    my $collect_home = $ENV{'GSDLCOLLECTDIR'};
     228    my $etc_dirname = &util::filename_cat($collect_home,"etc");
     229       
     230    my $build_dir = $self->{'build_dir'};
     231    my $idx_dirname = &util::filename_cat($build_dir,$ds_idx);
     232       
     233    $cgi_get_args .= "&instanceDir=$etc_dirname";
     234    $cgi_get_args .= "&dataDir=$idx_dirname";
     235
     236    $self->_admin_service($cgi_get_args);
     237}
     238
     239
    48240
    49241sub start
    … …  
    67259    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
    68260
    69 ##    print STDERR "**** server cmd = $server_java_cmd\n";
    70 
    71     my $pid = open(STARTIN,"-|");
    72 
    73     if ($pid==0) {
    74     # child process that will start up jetty and monitor output
    75 
    76     setpgrp(0,0);
     261##    print STDERR "**** server cmd start = $server_java_cmd\n";
     262
     263    my $server_status = "unknown";
     264
     265    my $server_already_running = $self->server_running();
     266
     267##    print STDERR "**** already running = $server_already_running\n";
     268
     269    if ($server_already_running) {
     270    $server_status = "already-running";
     271    }
     272    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
     273
     274    my $line;
     275    while (defined($line=<STARTIN>)) {
     276        # Scan through output until you see a line like:
     277        #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
     278        # which signifies that the server has started up and is
     279        # "ready and listening"
    77280   
    78     exec "$server_java_cmd 2>&1" || die "Failed to execute $server_java_cmd\n$!\n";
    79     # everything stops here
    80     }
    81 
    82     my $server_status = "unknown";
    83 
    84     my $line;
    85     while (defined($line=<STARTIN>)) {
    86     # Scan through output until you see a line like:
    87     #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
    88     # which signifies that the server has started up and is
    89     # "ready and listening"
    90    
    91 ##  print STDERR "**** $line";
     281##      print STDERR "**** $line";
    92282       
    93     # skip annoying "not listening" message
    94     next if ($line =~ m/WARN:\s*Not listening on monitor port/);
    95 
    96     if (($line =~ m/^(WARN|ERROR|SEVERE):/)
    97         || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
    98         print "Jetty startup: $line";
    99     }
    100    
    101    
    102     if ($line =~ m/WARN::failed SocketConnector/) {
    103         if ($line =~ m/Address already in use/) {
    104         $server_status = "already-running";
    105         }
    106         else {
    107         $server_status = "failed-to-start";
    108         }
    109         last;
    110     }
    111    
    112     if ($line =~ m/INFO::Started SocketConnector/) {
    113         $server_status = "explicitly-started";
    114         last;
    115     }
    116     }
     283        # skip annoying "not listening" message
     284        next if ($line =~ m/WARN:\s*Not listening on monitor port/);
     285
     286        if (($line =~ m/^(WARN|ERROR|SEVERE):/)
     287        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
     288        print "Jetty startup: $line";
     289        }
    117290       
     291        if ($line =~ m/WARN::failed SocketConnector/) {
     292        if ($line =~ m/Address already in use/) {
     293            $server_status = "already-running";
     294        }
     295        else {
     296            $server_status = "failed-to-start";
     297        }
     298        last;
     299        }
     300       
     301        if ($line =~ m/INFO::Started SocketConnector/) {
     302        $server_status = "explicitly-started";
     303        last;
     304        }
     305    }
     306    }
     307    else {
     308    print STDERR "Error: failed to start solr-jetty-server\n";
     309    print STDERR "$!\n";
     310    print STDERR "Command attempted was:\n";
     311    print STDERR "  $server_java_cmd\n";
     312    print STDERR "run from directory:\n";
     313    print STDERR "  $solr_home\n";
     314    print STDERR "----\n";
     315
     316    exit -1;
     317    }
     318
    118319    if ($server_status eq "explicitly-started") {
    119320    $self->{'jetty_explicitly_started'} = 1;
    120     print "Jetty server ready and listening for connections on port $jetty_server_port\n";
     321    print "Jetty server ready and listening for connections on port";
     322    print " $jetty_server_port\n";
    121323       
    122324    # now we know the server is ready to accept connections, fork a
    … …  
    126328    if (fork()==0) {
    127329        # child process
    128        
     330
    129331        my $line;
    130332        while (defined ($line = <STARTIN>)) {
    … …  
    155357    }
    156358    elsif ($server_status eq "already-running") {
    157     print "Using existing server detected on port $jetty_server_port\n";
     359    print STDERR "Using existing server detected on port $jetty_server_port\n";
    158360    $self->{'jetty_explicitly_started'} = 0;
    159    
    160     # Kill of the child process
    161 
    162     my $ks = kill(9,-$pid);
     361
     362    # silently stop our unneeded jetty server, using its unique key
     363
     364#   my $options = { 'do_wait' => 0, 'output_verbosity' => 2 };
     365
     366#   $self->stop($options);
    163367
    164368    # Consume any remaining (buffered) output (not interested in values)
    165     while (defined ($line = <STARTIN>)) {
     369#   my $line;
     370#   while (defined ($line = <STARTIN>)) {
    166371        # skip info lines
    167     }
    168     close(STARTIN);
    169     }
    170     else {
    171     print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n";
     372#   }
     373#   close(STARTIN);
     374    }
     375    elsif ($server_status eq "failed-to-start") {
     376    print STDERR "Started Solr/Jetty web server on port $jetty_server_port";
     377    print STDERR ", but encountered an initialization error\n";
    172378    exit -1;
    173379    }
    174380
    175 
    176 #    else {
    177 #   print STDERR "Error: failed to start solr-jetty-server\n";
    178 #   print STDERR "!$\n\n";
    179 #   print STDERR "Command attempted was:\n";
    180 #   print STDERR "  $server_java_cmd\n";
    181 #   print STDERR "run from directory:\n";
    182 #   print STDERR "  $solr_home\n";
    183 #   print STDERR "----\n";
    184 
    185 #   exit -1;
    186 #    }
    187 
    188 #    # If get to here then server started (and ready and listening)
    189 #    # *and* we are the parent process of the fork()
    190 
    191 }
    192 
     381}
    193382
    194383sub explicitly_started
    … …  
    198387    return $self->{'jetty_explicitly_started'};
    199388}
     389
    200390
    201391
    … …  
    228418    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
    229419    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
     420
     421##    print STDERR "**** java server stop cmd:\n  $server_java_cmd\n";
     422
    230423    if (open(STOPIN,"$server_java_cmd 2>&1 |")) {
    231424
    … …  
    246439    else {
    247440    print STDERR "Error: failed to stop solr-jetty-server\n";
    248     print STDERR "!$\n";
     441    print STDERR "$!\n";
    249442    print STDERR "Command attempted was:\n";
    250443    print STDERR "  $server_java_cmd\n";

  • gs3-extensions/solr/trunk/src/perllib/solrutil.pm

    r24453 r24483  
    6363sub open_post_pipe
    6464{
    65     my ($collect,$doc_tag_level) = @_;
     65    my ($collect,$ds_idx) = @_;
    6666
    6767    my $search_path = get_search_path();
    … …  
    7575   
    7676    # Now run solr-post command
    77     my $core = $collect."-".lc($doc_tag_level);
     77    my $core = $collect."-".$ds_idx;
    7878    my $post_props = "-Durl=http://localhost:$jetty_port/solr/$core/update";
    7979    $post_props .= " -Ddata=stdin";
Note: See TracChangeset for help on using the changeset viewer.