Changeset 24453

Show
Ignore:
Timestamp:
23.08.2011 14:23:40 (8 years ago)
Author:
davidb
Message:

Tidy up of code. Better structuring of classes

Location:
gs3-extensions/solr/trunk/src
Files:
2 added
2 modified

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24447 r24453  
    4141    unshift (@INC, "$ENV{'GSDLHOME'}/perllib"); 
    4242    die "GEXT_SOLR not set\n" unless defined $ENV{'GEXT_SOLR'}; 
     43 
     44    my $solr_ext = $ENV{'GEXT_SOLR'}; 
     45    unshift (@INC, "$solr_ext/perllib"); 
    4346} 
    4447 
    4548use strict; 
    4649use util; 
     50use solrutil; 
     51use solrserver; 
     52 
    4753 
    4854# Not quite OO, but close enough for now 
    4955# 
    50 my $self = { 'full_server_jar' => undef, 
    51          'jetty_explicitly_started' => undef, 
    52          'jetty_stop_key' => "greenstone-solr" 
    53          }; 
    54  
    55  
    56  
    57 sub locate_file 
    58 { 
    59     my ($search_path,$suffix) = @_; 
    60          
    61     foreach my $sp (@$search_path) { 
    62     my $full_path = &util::filename_cat($sp,$suffix); 
    63      
    64     if (-f $full_path) { 
    65         return $full_path; 
    66     } 
    67     } 
    68      
    69     # if get to here, then failed to find match 
    70  
    71     print STDERR "Error: Failed to find '$suffix'\n"; 
    72     print STDERR "  Looked in: ", join(", ", @$search_path), "\n"; 
    73     exit -1; 
    74 } 
    75  
    76 sub start_solr_server 
    77 { 
    78     my ($search_path) = @_; 
    79  
    80     my $solr_home         = $ENV{'GEXT_SOLR'}; 
    81     my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'}; 
    82     my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
    83  
    84     chdir($solr_home); 
    85      
    86     my $solr_etc = &util::filename_cat($solr_home,"etc"); 
    87  
    88     my $server_props = "-DSTOP.PORT=$jetty_stop_port"; 
    89     $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    90     $server_props .= " -Dsolr.solr.home=$solr_etc"; 
    91  
    92     my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar"); 
    93     my $full_server_jar = locate_file($search_path,$server_jar); 
    94     $self->{'full_server_jar'} = $full_server_jar; 
    95      
    96     my $server_java_cmd = "java $server_props -jar \"$full_server_jar\""; 
    97  
    98 ##    print STDERR "**** server cmd = $server_java_cmd\n"; 
    99  
    100     if (open(SIN,"$server_java_cmd 2>&1 |")) { 
    101      
    102     my $server_status = "unknown"; 
    103  
    104     my $line; 
    105     while (defined($line=<SIN>)) { 
    106         # Scan through output until you see a line like: 
    107         #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983 
    108         # which signifies that the server has started up and is 
    109         # "ready and listening" 
    110  
    111 ##      print STDERR "**** $line"; 
    112  
    113         if (($line =~ m/^(WARN|ERROR|SEVERE):/) 
    114         || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) { 
    115         print $line; 
    116         } 
    117  
    118  
    119         if ($line =~ m/WARN::failed SocketConnector/) { 
    120         if ($line =~ m/Address already in use/) { 
    121             $server_status = "already-running"; 
    122         } 
    123         else { 
    124             $server_status = "failed-to-start"; 
    125         } 
    126         last; 
    127         } 
    128          
    129         if ($line =~ m/INFO::Started SocketConnector/) { 
    130         $server_status = "explicitly-started"; 
    131         last; 
    132         } 
    133     } 
    134  
    135     if ($server_status eq "explicitly-started") { 
    136         $self->{'jetty_explicitly_started'} = 1; 
    137         print STDERR "Jetty server ready and listening for connections\n"; 
    138     } 
    139     elsif ($server_status eq "already-running") { 
    140         print STDERR "Using existing server detected on port $jetty_server_port\n"; 
    141     } 
    142     else { 
    143         print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n"; 
    144         exit -1; 
    145     } 
    146          
    147     # now we know the server is ready to accept connections, fork a 
    148     # child process that continues to listen to the output and 
    149     # prints out any lines that are not INFO lines 
    150  
    151     if (fork()==0) { 
    152         # child process 
    153          
    154         my $line; 
    155         while (defined ($line = <SIN>)) { 
    156         next if ($line =~ m/^INFO:/); 
    157         next if ($line =~ m/^[0-9 :-]*INFO::/); 
    158         next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);  
    159         } 
    160         close(SIN); 
    161          
    162         # And now stop nicely 
    163         exit 0; 
    164     } 
    165     } 
    166     else { 
    167     print STDERR "Error: failed to start solr-jetty-server\n"; 
    168     print STDERR "!$\n\n"; 
    169     print STDERR "Command attempted was:\n"; 
    170     print STDERR "  $server_java_cmd\n"; 
    171     print STDERR "run from directory:\n"; 
    172     print STDERR "  $solr_home\n"; 
    173     print STDERR "----\n"; 
    174  
    175     exit -1; 
    176     } 
    177  
    178     # If get to here then server started (and ready and listening) 
    179     # *and* we are the parent process of the fork() 
    180  
    181 } 
    182  
    183  
    184  
    185 sub stop_solr_server 
    186 { 
    187     my $full_server_jar = $self->{'full_server_jar'}; 
    188     my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'}; 
    189      
    190     my $server_props = "-DSTOP.PORT=$jetty_stop_port"; 
    191     $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    192     my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop"; 
    193  
    194     my $server_status = system($server_java_cmd); 
    195      
    196     if ($server_status!=0) { 
    197     print STDERR "Error: failed to stop solr-jetty-server\n"; 
    198     print STDERR "!$\n"; 
    199     exit -1; 
    200     } 
    201     else { 
    202     wait(); # let the child process finish 
    203     print STDERR "Jetty server shutdown\n"; 
    204     } 
    205 } 
    206  
     56my $self = { 'solr_server' => undef }; 
    20757 
    20858sub open_java_solr 
    20959{ 
    21060  my ($collect, $doc_tag_level,$full_builddir,$indexdir,$removeold) = @_; 
    211  
    21261 
    21362  # if removeold set, then delete the curring $full_builddir 
     
    21766  } 
    21867 
    219   my $search_path = []; 
    220  
    221   push(@$search_path,$ENV{'GSDLCOLLECTDIR'}) if defined $ENV{'GSDLCOLLECTDIR'}; 
    222   push(@$search_path,$ENV{'GSDLHOME'})       if defined $ENV{'GSDLHOME'}; 
    223   push(@$search_path,$ENV{'GEXT_SOLR'})      if defined $ENV{'GEXT_SOLR'}; 
    224  
    225  
    226   # The following returns once Jetty has generated its  
    227   # "reading and listening" line 
    228   # 
    229   start_solr_server($search_path); 
    230  
    231   # Now run the solr-post command 
    232  
    233   chdir($ENV{'GEXT_SOLR'}); 
     68  # If the Solr/Jetty server is not already running, the following starts 
     69  # it up, and only returns when the server is "reading and listening" 
    23470   
    235   my $post_jar   = &util::filename_cat("lib","java","solr-post.jar"); 
    236   my $full_post_jar   = locate_file($search_path,$post_jar); 
    237  
    238   my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
    239  
    240   # Now run solr-post command 
    241   my $post_props = "-Durl=http://localhost:$jetty_server_port/solr/$collect-$doc_tag_level/update"; 
    242   $post_props .= " -Ddata=stdin"; 
    243   $post_props .= " -Dcommit=yes"; 
    244  
    245   my $post_java_cmd = "java $post_props -jar \"$full_post_jar\""; 
    246  
    247 ###  print STDERR "**** post cmd = $post_java_cmd\n"; 
    248    
    249   open (PIPEOUT, "| $post_java_cmd")  
    250       || die "Error in solr_passes.pl: Failed to run $post_java_cmd\n!$\n"; 
    251 } 
    252  
    253  
     71  my $solr_server = new solrserver(); 
     72  $solr_server->start(); 
     73  $self->{'solr_server'} = $solr_server; 
     74 
     75  # Now start up the solr-post command 
     76  &solrutil::open_post_pipe($collect,$doc_tag_level); 
     77} 
    25478 
    25579sub close_java_solr 
    25680{ 
    257     # closing the pipe has the effect of shutting down solr-post.jar 
    258     close(PIPEOUT); 
    259      
    260     if ($self->{'jetty_explicitly_started'}) { 
    261     stop_solr_server(); 
    262     } 
    263 } 
    264  
     81    &solrutil::close_post_pipe(); 
     82       
     83    my $solr_server = $self->{'solr_server'}; 
     84    if ($solr_server->explicitly_started()) { 
     85    $solr_server->stop(); 
     86    } 
     87} 
    26588 
    26689#---- 
     
    348171    my $line; 
    349172    while (defined ($line = <STDIN>)) { 
    350     print PIPEOUT $line; 
     173    &solrutil::print_to_post_pipe($line); 
    351174    } 
    352175} 
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24447 r24453  
    3131 
    3232use lucenebuilder; 
     33use solrserver; 
    3334use Config; # for getting the perlpath in the recommended way 
    3435 
     
    271272 
    272273 
     274sub solr_core_admin 
     275{ 
     276    my $self = shift (@_); 
     277    my ($url) = @_; 
     278 
     279    my $cmd = "wget -q -O - \"$url\""; 
     280 
     281    my $status = undef; 
     282 
     283    if (open(WIN,"$cmd |")) { 
     284 
     285    my $xml_output = ""; 
     286    my $line; 
     287    while (defined ($line=<WIN>)) { 
     288 
     289        $xml_output .= $line; 
     290    } 
     291    close(WIN); 
     292 
     293##  print $xml_output; 
     294 
     295    ($status) = ($xml_output =~ m!<int name="status">(\d+)</int>!s); 
     296     
     297    } 
     298    else { 
     299    print STDERR "Warning: failed to run $cmd\n"; 
     300    print STDERR "  $!\n"; 
     301    } 
     302 
     303    return $status; 
     304 
     305} 
     306 
    273307sub pre_build_indexes 
    274308{ 
     
    277311    my $outhandle = $self->{'outhandle'}; 
    278312 
    279     # read in build.cfg if in incremental mode??? 
     313    # If the Solr/Jetty server is not already running, the following starts 
     314    # it up, and only returns when the server is "reading and listening" 
     315   
     316    my $solr_server = new solrserver(); 
     317    $solr_server->start(); 
     318    $self->{'solr_server'} = $solr_server; 
    280319 
    281320    my $indexes = []; 
     
    350389    } 
    351390 
    352     # write out solr 'schema.xml' (and related) file  
     391    # Write out solr 'schema.xml' (and related) file  
     392    # 
    353393    $self->make_final_field_list(); 
    354394    $self->premake_solr_auxiliary_files(); 
    355395 
    356     # if collect==core not already in solr.xml (check with STATUS) 
    357     # => use CREATE API to add to solr.xml 
    358     # 
    359     # else  
    360     # => use RELOAD call to refresh fields now expressed in schema.xml 
     396    # Now update the solr-core information in solr.xml 
     397    # => at most two cores <colname>-Doc and <colname>-Sec 
     398 
     399    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
     400    my $base_url = "http://localhost:$jetty_server_port/solr/admin/cores"; 
     401 
     402    my $collection = $self->{'collection'}; 
     403 
     404    foreach my $level (keys %{$self->{'levels'}}) { 
     405 
     406    my $llevel = $mgppbuilder::level_map{$level}; 
     407 
     408    my $core = $collection."-".lc($llevel); 
     409    my $check_core_url = "$base_url?action=STATUS&core=$core"; 
     410 
     411    my $check_status = $self->solr_core_admin($check_core_url); 
     412     
     413    print STDERR "*** check status = $check_status\n"; 
     414 
     415 
     416    # if collect==core not already in solr.xml (check with STATUS) 
     417    # => use CREATE API to add to solr.xml 
     418    # 
     419    # else  
     420    # => use RELOAD call to refresh fields now expressed in schema.xml 
     421    } 
    361422 
    362423} 
     
    499560    #  $self->make_final_field_list() 
    500561    # as this has been done in our pre_build_indexes() phase for solr 
     562 
     563 
     564    # Also need to stop the Solr/jetty server if it was explicitly started 
     565    # in pre_build_indexes() 
    501566     
     567    my $solr_server = $self->{'solr_server'}; 
     568 
     569    if ($solr_server->explicitly_started()) { 
     570    $solr_server->stop(); 
     571    } 
     572 
     573    $self->{'solr_server'} = undef; 
     574 
    502575}     
    503576