Changeset 24483 for gs3-extensions

Show
Ignore:
Timestamp:
26.08.2011 00:22:29 (8 years ago)
Author:
davidb
Message:

Reworking of code that detects existing running instance of Solr/Jetty server. This was due to Windows version of Perl not implementing '|-' on an open call. Code currently messy and needs a further tidy up.

Location:
gs3-extensions/solr/trunk/src
Files:
4 modified

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24453 r24483  
    5858sub open_java_solr 
    5959{ 
    60   my ($collect, $doc_tag_level,$full_builddir,$indexdir,$removeold) = @_; 
     60  my ($collect, $ds_idx,$full_builddir,$indexdir,$removeold) = @_; 
    6161 
    6262  # if removeold set, then delete the curring $full_builddir 
     
    7474 
    7575  # Now start up the solr-post command 
    76   &solrutil::open_post_pipe($collect,$doc_tag_level); 
     76  &solrutil::open_post_pipe($collect,$ds_idx); 
    7777} 
    7878 
     
    224224 
    225225  if ($filtered_argc < 5) { 
    226     print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" doc-tag-level build-dir index-name\n"; 
     226    print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" {d|s}idx build-dir index-name\n"; 
    227227    exit 1; 
    228228  } 
     
    230230  my $collect       = $filtered_argv[0]; 
    231231  my $mode          = $filtered_argv[1]; 
    232   my $doc_tag_level = $filtered_argv[2]; 
     232  my $ds_idx        = $filtered_argv[2]; 
    233233  my $full_builddir = $filtered_argv[3]; 
    234234  my $indexdir      = $filtered_argv[4]; 
     
    237237  # documents, not if we are just storing the text 
    238238  if ($mode eq "index") { 
    239     open_java_solr($collect, $doc_tag_level, $full_builddir, $indexdir, $removeold); 
     239    open_java_solr($collect, $ds_idx, $full_builddir, $indexdir, $removeold); 
    240240  } 
    241241 
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24456 r24483  
    117117        print STDERR "Sections:      $solr_passes_sections\n"; 
    118118        print STDERR "Build Dir:     $build_dir\n"; 
    119         print STDERR "Cmd:           $solr_passes_exe $collection text $solr_passes_sections \"$build_dir\" \"dummy\"   $osextra\n"; 
    120     if (!open($handle, "| $solr_passes_exe $collection text $solr_passes_sections \"$build_dir\" \"dummy\"   $osextra")) 
     119        print STDERR "Cmd:           $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra\n"; 
     120    if (!open($handle, "| $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra")) 
    121121    { 
    122122        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'}; 
     
    272272 
    273273 
    274 sub solr_core_admin 
    275 { 
    276     my $self = shift (@_); 
    277     my ($url) = @_; 
    278  
    279     my $cmd = "wget -O - \"$url\" 2>&1"; 
    280  
    281     my $preamble_output = "";     
    282     my $xml_output = ""; 
    283     my $error_output = undef; 
    284  
    285     my $in_preamble = 1; 
    286      
    287     if (open(WIN,"$cmd |")) { 
    288  
    289     my $line; 
    290     while (defined ($line=<WIN>)) { 
    291  
    292         if ($line =~ m/ERROR \d+:/) { 
    293         chomp $line; 
    294         $error_output = $line; 
    295         last; 
    296         } 
    297         elsif ($in_preamble) { 
    298         if ($line =~ m/<.*>/) { 
    299             $in_preamble = 0; 
    300         } 
    301         else { 
    302             $preamble_output .= $line; 
    303         } 
    304         } 
    305  
    306         if (!$in_preamble) { 
    307         $xml_output .= $line; 
    308         } 
    309     } 
    310     close(WIN); 
    311  
    312     } 
    313     else { 
    314     $error_output = "Error: failed to run $cmd\n"; 
    315     $error_output .= "  $!\n"; 
    316     } 
    317  
    318     my $output = { 'preamble' => $preamble_output, 
    319            'output'   => $xml_output, 
    320            'error'    => $error_output }; 
    321  
    322     return $output; 
    323 } 
    324  
    325274sub pre_build_indexes 
    326275{ 
     
    415364    # => at most two cores <colname>-Doc and <colname>-Sec 
    416365 
    417     my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
    418     my $base_url = "http://localhost:$jetty_server_port/solr/admin/cores"; 
    419  
    420366    my $collection = $self->{'collection'}; 
    421367 
     
    427373    my ($pindex) = $level =~ /^(.)/; 
    428374     
    429     my $llevel = $mgppbuilder::level_map{$level}; 
    430     my $core = $collection."-".lc($llevel); 
    431      
    432      
    433     # if collect==core not already in solr.xml (check with STATUS) 
    434     # => use CREATE API to add to solr.xml 
     375##  my $llevel = $mgppbuilder::level_map{$level}; 
     376##  my $core = $collection."-".lc($llevel); 
     377         
     378    my $core = $collection."-".$pindex.$idx; 
     379 
     380    # if collect==core already in solr.xml (check with STATUS) 
     381    # => use RELOAD call to refresh fields now expressed in schema.xml 
    435382    # 
    436383    # else  
    437     # => use RELOAD call to refresh fields now expressed in schema.xml 
    438      
    439      
    440     my $check_core_url    = "$base_url?action=STATUS&core=$core"; 
    441     my $output = $self->solr_core_admin($check_core_url); 
    442  
    443     if (defined $output->{'error'}) { 
    444  
    445         my $preamble = $output->{'preamble'}; 
    446         my $error    = $output->{'error'}; 
    447  
    448         print STDERR "----\n"; 
    449         print STDERR "Error: Failed to get XML response from:\n"; 
    450         print STDERR "         $check_core_url\n"; 
    451         print STDERR "Output was:\n"; 
    452         print STDERR $preamble if ($preamble ne ""); 
    453         print STDERR "$error\n"; 
    454         print STDERR "----\n"; 
    455  
    456         next; 
    457     } 
    458      
    459     # If the collection doesn't exist yet, then there will be 
    460     # an empty element of the form: 
    461     #   <lst name="collect-doc"/> 
    462     # where 'collect' is the actual name of the collection,  
    463     # such as demo 
    464  
    465     my $xml_output = $output->{'output'}; 
    466      
    467     my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>"; 
    468      
    469     my $check_core_exists = !($xml_output =~ m/$empty_element/s); 
    470      
    471     if ($check_core_exists) { 
    472          
    473         my $reload_core_url    = "$base_url?action=RELOAD&core=$core"; 
    474          
     384    # => use CREATE API to add to solr.xml 
     385         
     386    my $check_core_exists = $solr_server->admin_ping_core($core); 
     387        
     388    if ($check_core_exists) {        
    475389        print $outhandle "Reloading Solr core: $core\n"; 
    476         $self->solr_core_admin($reload_core_url); 
     390        $solr_server->admin_reload_core($core); 
    477391    } 
    478392    else { 
    479          
    480         my $collect_home = $ENV{'GSDLCOLLECTDIR'}; 
    481         my $etc_dirname = &util::filename_cat($collect_home,"etc"); 
    482          
    483         my $build_dir = $self->{'build_dir'}; 
    484         my $idx_dirname = &util::filename_cat($build_dir,$pindex.$idx); 
    485          
    486         my $create_core_url    = "$base_url?action=CREATE&name=$core"; 
    487         $create_core_url .= "&instanceDir=$etc_dirname"; 
    488         $create_core_url .= "&dataDir=$idx_dirname"; 
    489          
    490393        print $outhandle "Creating Solr core: $core\n"; 
    491         $self->solr_core_admin($create_core_url); 
     394        $solr_server->admin_create_core($core); 
    492395    } 
    493396    } 
     
    574477    } else { 
    575478    my $collection = $self->{'collection'}; 
    576  
    577     print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $solr_passes_sections \"$build_dir\" \"$indexdir\"   $osextra\n"; 
    578     if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $solr_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) { 
     479    my $ds_idx = $self->{'index_mapping'}->{$index}; 
     480 
     481    print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra\n"; 
     482    if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra")) { 
    579483        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    580484        die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n"; 
  • gs3-extensions/solr/trunk/src/perllib/solrserver.pm

    r24453 r24483  
    2828 
    2929use strict;  
     30#no strict 'refs'; 
    3031 
    3132use solrutil; 
     33 
     34my $key_count = 0; 
    3235 
    3336sub new { 
    3437    my $class = shift(@_); 
    3538 
    36     my $self = { 'jetty_stop_key' => "greenstone-solr" }; 
     39    $key_count++; 
     40    my $self = { 'jetty_stop_key' => "greenstone-solr-".$$."-".$key_count }; 
    3741 
    3842    my $search_path = &solrutil::get_search_path(); 
     
    4448    $self->{'jetty_explicitly_started'} = undef; 
    4549 
     50    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
     51    my $base_url = "http://localhost:$jetty_server_port/solr/"; 
     52    my $admin_url = "http://localhost:$jetty_server_port/solr/admin/cores"; 
     53     
     54    $self->{'base-url'} = $base_url; 
     55    $self->{'admin-url'} = $admin_url; 
     56 
    4657    return bless $self, $class; 
    4758} 
     59 
     60 
     61 
     62sub _wget_service 
     63{ 
     64    my $self = shift (@_); 
     65    my ($output_format,$url,$cgi_get_args) = @_; 
     66 
     67    my $full_url = $url; 
     68 
     69    $url .= "?$cgi_get_args" if (defined $cgi_get_args); 
     70     
     71    my $cmd = "wget -O - \"$url\" 2>&1"; 
     72 
     73##    print STDERR "**** wget cmd: $cmd\n"; 
     74 
     75    my $preamble_output = "";     
     76    my $xml_output = ""; 
     77    my $error_output = undef; 
     78 
     79    my $in_preamble = ($output_format eq "xml") ? 1 : 0; 
     80     
     81    if (open(WIN,"$cmd |")) { 
     82 
     83    my $line; 
     84    while (defined ($line=<WIN>)) { 
     85 
     86        if ($line =~ m/ERROR \d+:/) { 
     87        chomp $line; 
     88        $error_output = $line; 
     89        last; 
     90        } 
     91        elsif ($line =~ m/failed: Connection refused/) { 
     92        chomp $line; 
     93        $error_output = $line; 
     94        last; 
     95        } 
     96        elsif ($in_preamble) { 
     97        if ($line =~ m/<.*>/) { 
     98            $in_preamble = 0; 
     99        } 
     100        else { 
     101            $preamble_output .= $line; 
     102        } 
     103        } 
     104 
     105        if (! $in_preamble) { 
     106        $xml_output .= $line; 
     107        } 
     108    } 
     109    close(WIN); 
     110 
     111    } 
     112    else { 
     113    $error_output = "Error: failed to run $cmd\n"; 
     114    $error_output .= "  $!\n"; 
     115    } 
     116 
     117    my $output = { 'preamble' => $preamble_output, 
     118           'output'   => $xml_output, 
     119           'error'    => $error_output }; 
     120 
     121    return $output; 
     122} 
     123 
     124 
     125sub _base_service 
     126{ 
     127    my $self = shift (@_); 
     128    my ($cgi_get_args) = @_; 
     129 
     130    my $base_url = $self->{'base-url'}; 
     131 
     132    return $self->_wget_service("html",$base_url,$cgi_get_args); 
     133} 
     134  
     135sub _admin_service 
     136{ 
     137    my $self = shift (@_); 
     138    my ($cgi_get_args) = @_; 
     139 
     140    my $admin_url = $self->{'admin-url'}; 
     141 
     142    return $self->_wget_service("xml",$admin_url,$cgi_get_args); 
     143} 
     144 
     145 
     146sub server_running 
     147{ 
     148    my $self = shift @_; 
     149 
     150    my $output = $self->_base_service(); 
     151 
     152    my $have_error = defined $output->{'error'}; 
     153 
     154    my $running = !$have_error; 
     155 
     156    return $running; 
     157} 
     158 
     159 
     160sub admin_ping_core 
     161{ 
     162    my $self = shift @_; 
     163    my ($core) = @_; 
     164 
     165    my $cgi_get_args = "action=STATUS&core=$core"; 
     166 
     167    my $ping_status = 1; 
     168 
     169    my $output = $self->_admin_service($cgi_get_args); 
     170 
     171    if (defined $output->{'error'}) { 
     172    # severe error, such as failing to connect to the server 
     173    $ping_status = 0; 
     174 
     175    my $url      = $output->{'url'}; 
     176    my $preamble = $output->{'preamble'}; 
     177    my $error    = $output->{'error'}; 
     178     
     179    print STDERR "----\n"; 
     180    print STDERR "Error: Failed to get XML response from:\n"; 
     181    print STDERR "         $url\n"; 
     182    print STDERR "Output was:\n"; 
     183    print STDERR $preamble if ($preamble ne ""); 
     184    print STDERR "$error\n"; 
     185    print STDERR "----\n"; 
     186    } 
     187    else { 
     188     
     189    # If the collection doesn't exist yet, then there will be 
     190    # an empty element of the form: 
     191    #   <lst name="collect-doc"/> 
     192    # where 'collect' is the actual name of the collection,  
     193    # such as demo 
     194 
     195    my $xml_output = $output->{'output'}; 
     196     
     197    my $empty_element="<lst\\s+name=\"$core\"\\s*\\/>"; 
     198     
     199    $ping_status = !($xml_output =~ m/$empty_element/s); 
     200    } 
     201 
     202 
     203    return $ping_status; 
     204} 
     205 
     206 
     207sub admin_reload_core 
     208{ 
     209    my $self = shift @_; 
     210    my ($core) = @_; 
     211 
     212    my $cgi_get_args = "action=RELOAD&core=$core"; 
     213 
     214    $self->_admin_service($cgi_get_args); 
     215} 
     216 
     217 
     218sub admin_create_core 
     219{ 
     220    my $self = shift @_; 
     221    my ($core) = @_; 
     222 
     223    my ($ds_idx) = ($core =~ m/^.*-(.*)$/); 
     224 
     225    my $cgi_get_args = "action=CREATE&name=$core"; 
     226 
     227    my $collect_home = $ENV{'GSDLCOLLECTDIR'}; 
     228    my $etc_dirname = &util::filename_cat($collect_home,"etc"); 
     229         
     230    my $build_dir = $self->{'build_dir'}; 
     231    my $idx_dirname = &util::filename_cat($build_dir,$ds_idx); 
     232         
     233    $cgi_get_args .= "&instanceDir=$etc_dirname"; 
     234    $cgi_get_args .= "&dataDir=$idx_dirname"; 
     235 
     236    $self->_admin_service($cgi_get_args); 
     237} 
     238 
     239 
    48240 
    49241sub start 
     
    67259    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\""; 
    68260 
    69 ##    print STDERR "**** server cmd = $server_java_cmd\n"; 
    70  
    71     my $pid = open(STARTIN,"-|"); 
    72  
    73     if ($pid==0) { 
    74     # child process that will start up jetty and monitor output 
    75  
    76     setpgrp(0,0); 
     261##    print STDERR "**** server cmd start = $server_java_cmd\n"; 
     262 
     263    my $server_status = "unknown"; 
     264 
     265    my $server_already_running = $self->server_running(); 
     266 
     267##    print STDERR "**** already running = $server_already_running\n"; 
     268 
     269    if ($server_already_running) { 
     270    $server_status = "already-running"; 
     271    } 
     272    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) { 
     273 
     274    my $line; 
     275    while (defined($line=<STARTIN>)) { 
     276        # Scan through output until you see a line like: 
     277        #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983 
     278        # which signifies that the server has started up and is 
     279        # "ready and listening" 
    77280     
    78     exec "$server_java_cmd 2>&1" || die "Failed to execute $server_java_cmd\n$!\n"; 
    79     # everything stops here 
    80     } 
    81  
    82     my $server_status = "unknown"; 
    83  
    84     my $line; 
    85     while (defined($line=<STARTIN>)) { 
    86     # Scan through output until you see a line like: 
    87     #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983 
    88     # which signifies that the server has started up and is 
    89     # "ready and listening" 
    90      
    91 ##  print STDERR "**** $line"; 
     281##      print STDERR "**** $line"; 
    92282         
    93     # skip annoying "not listening" message 
    94     next if ($line =~ m/WARN:\s*Not listening on monitor port/); 
    95  
    96     if (($line =~ m/^(WARN|ERROR|SEVERE):/) 
    97         || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) { 
    98         print "Jetty startup: $line"; 
    99     } 
    100      
    101      
    102     if ($line =~ m/WARN::failed SocketConnector/) { 
    103         if ($line =~ m/Address already in use/) { 
    104         $server_status = "already-running"; 
    105         } 
    106         else { 
    107         $server_status = "failed-to-start"; 
    108         } 
    109         last; 
    110     } 
    111      
    112     if ($line =~ m/INFO::Started SocketConnector/) { 
    113         $server_status = "explicitly-started"; 
    114         last; 
    115     } 
    116     } 
     283        # skip annoying "not listening" message 
     284        next if ($line =~ m/WARN:\s*Not listening on monitor port/); 
     285 
     286        if (($line =~ m/^(WARN|ERROR|SEVERE):/) 
     287        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) { 
     288        print "Jetty startup: $line"; 
     289        } 
    117290         
     291        if ($line =~ m/WARN::failed SocketConnector/) { 
     292        if ($line =~ m/Address already in use/) { 
     293            $server_status = "already-running"; 
     294        } 
     295        else { 
     296            $server_status = "failed-to-start"; 
     297        } 
     298        last; 
     299        } 
     300         
     301        if ($line =~ m/INFO::Started SocketConnector/) { 
     302        $server_status = "explicitly-started"; 
     303        last; 
     304        } 
     305    } 
     306    } 
     307    else { 
     308    print STDERR "Error: failed to start solr-jetty-server\n"; 
     309    print STDERR "$!\n"; 
     310    print STDERR "Command attempted was:\n"; 
     311    print STDERR "  $server_java_cmd\n"; 
     312    print STDERR "run from directory:\n"; 
     313    print STDERR "  $solr_home\n"; 
     314    print STDERR "----\n"; 
     315 
     316    exit -1; 
     317    } 
     318 
    118319    if ($server_status eq "explicitly-started") { 
    119320    $self->{'jetty_explicitly_started'} = 1; 
    120     print "Jetty server ready and listening for connections on port $jetty_server_port\n"; 
     321    print "Jetty server ready and listening for connections on port"; 
     322    print " $jetty_server_port\n"; 
    121323         
    122324    # now we know the server is ready to accept connections, fork a 
     
    126328    if (fork()==0) { 
    127329        # child process 
    128          
     330 
    129331        my $line; 
    130332        while (defined ($line = <STARTIN>)) { 
     
    155357    } 
    156358    elsif ($server_status eq "already-running") { 
    157     print "Using existing server detected on port $jetty_server_port\n"; 
     359    print STDERR "Using existing server detected on port $jetty_server_port\n"; 
    158360    $self->{'jetty_explicitly_started'} = 0; 
    159      
    160     # Kill of the child process 
    161  
    162     my $ks = kill(9,-$pid); 
     361 
     362    # silently stop our unneeded jetty server, using its unique key 
     363 
     364#   my $options = { 'do_wait' => 0, 'output_verbosity' => 2 }; 
     365 
     366#   $self->stop($options); 
    163367 
    164368    # Consume any remaining (buffered) output (not interested in values) 
    165     while (defined ($line = <STARTIN>)) {  
     369#   my $line; 
     370#   while (defined ($line = <STARTIN>)) {  
    166371        # skip info lines 
    167     } 
    168     close(STARTIN); 
    169     } 
    170     else { 
    171     print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n"; 
     372#   } 
     373#   close(STARTIN); 
     374    } 
     375    elsif ($server_status eq "failed-to-start") { 
     376    print STDERR "Started Solr/Jetty web server on port $jetty_server_port"; 
     377    print STDERR ", but encountered an initialization error\n"; 
    172378    exit -1; 
    173379    } 
    174380 
    175  
    176 #    else { 
    177 #   print STDERR "Error: failed to start solr-jetty-server\n"; 
    178 #   print STDERR "!$\n\n"; 
    179 #   print STDERR "Command attempted was:\n"; 
    180 #   print STDERR "  $server_java_cmd\n"; 
    181 #   print STDERR "run from directory:\n"; 
    182 #   print STDERR "  $solr_home\n"; 
    183 #   print STDERR "----\n"; 
    184  
    185 #   exit -1; 
    186 #    } 
    187  
    188 #    # If get to here then server started (and ready and listening) 
    189 #    # *and* we are the parent process of the fork() 
    190  
    191 } 
    192  
     381} 
    193382 
    194383sub explicitly_started 
     
    198387    return $self->{'jetty_explicitly_started'}; 
    199388} 
     389 
    200390 
    201391 
     
    228418    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    229419    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop"; 
     420 
     421##    print STDERR "**** java server stop cmd:\n  $server_java_cmd\n"; 
     422 
    230423    if (open(STOPIN,"$server_java_cmd 2>&1 |")) { 
    231424 
     
    246439    else { 
    247440    print STDERR "Error: failed to stop solr-jetty-server\n"; 
    248     print STDERR "!$\n"; 
     441    print STDERR "$!\n"; 
    249442    print STDERR "Command attempted was:\n"; 
    250443    print STDERR "  $server_java_cmd\n"; 
  • gs3-extensions/solr/trunk/src/perllib/solrutil.pm

    r24453 r24483  
    6363sub open_post_pipe 
    6464{ 
    65     my ($collect,$doc_tag_level) = @_; 
     65    my ($collect,$ds_idx) = @_; 
    6666 
    6767    my $search_path = get_search_path(); 
     
    7575     
    7676    # Now run solr-post command 
    77     my $core = $collect."-".lc($doc_tag_level); 
     77    my $core = $collect."-".$ds_idx; 
    7878    my $post_props = "-Durl=http://localhost:$jetty_port/solr/$core/update"; 
    7979    $post_props .= " -Ddata=stdin";