Changeset 24453


Ignore:
Timestamp:
2011-08-23T14:23:40+12:00 (11 years ago)
Author:
davidb
Message:

Tidy up of code. Better structuring of classes

Location:
gs3-extensions/solr/trunk/src
Files:
2 added
2 edited

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24447 r24453  
    4141    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
    4242    die "GEXT_SOLR not set\n" unless defined $ENV{'GEXT_SOLR'};
     43
     44    my $solr_ext = $ENV{'GEXT_SOLR'};
     45    unshift (@INC, "$solr_ext/perllib");
    4346}
    4447
    4548use strict;
    4649use util;
     50use solrutil;
     51use solrserver;
     52
    4753
    4854# Not quite OO, but close enough for now
    4955#
    50 my $self = { 'full_server_jar' => undef,
    51          'jetty_explicitly_started' => undef,
    52          'jetty_stop_key' => "greenstone-solr"
    53          };
    54 
    55 
    56 
    57 sub locate_file
    58 {
    59     my ($search_path,$suffix) = @_;
    60        
    61     foreach my $sp (@$search_path) {
    62     my $full_path = &util::filename_cat($sp,$suffix);
    63    
    64     if (-f $full_path) {
    65         return $full_path;
    66     }
    67     }
    68    
    69     # if get to here, then failed to find match
    70 
    71     print STDERR "Error: Failed to find '$suffix'\n";
    72     print STDERR "  Looked in: ", join(", ", @$search_path), "\n";
    73     exit -1;
    74 }
    75 
    76 sub start_solr_server
    77 {
    78     my ($search_path) = @_;
    79 
    80     my $solr_home         = $ENV{'GEXT_SOLR'};
    81     my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'};
    82     my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
    83 
    84     chdir($solr_home);
    85    
    86     my $solr_etc = &util::filename_cat($solr_home,"etc");
    87 
    88     my $server_props = "-DSTOP.PORT=$jetty_stop_port";
    89     $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
    90     $server_props .= " -Dsolr.solr.home=$solr_etc";
    91 
    92     my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
    93     my $full_server_jar = locate_file($search_path,$server_jar);
    94     $self->{'full_server_jar'} = $full_server_jar;
    95    
    96     my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
    97 
    98 ##    print STDERR "**** server cmd = $server_java_cmd\n";
    99 
    100     if (open(SIN,"$server_java_cmd 2>&1 |")) {
    101    
    102     my $server_status = "unknown";
    103 
    104     my $line;
    105     while (defined($line=<SIN>)) {
    106         # Scan through output until you see a line like:
    107         #   2011-08-22 .. :INFO::Started SocketConnector@0.0.0.0:8983
    108         # which signifies that the server has started up and is
    109         # "ready and listening"
    110 
    111 ##      print STDERR "**** $line";
    112 
    113         if (($line =~ m/^(WARN|ERROR|SEVERE):/)
    114         || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
    115         print $line;
    116         }
    117 
    118 
    119         if ($line =~ m/WARN::failed SocketConnector/) {
    120         if ($line =~ m/Address already in use/) {
    121             $server_status = "already-running";
    122         }
    123         else {
    124             $server_status = "failed-to-start";
    125         }
    126         last;
    127         }
    128        
    129         if ($line =~ m/INFO::Started SocketConnector/) {
    130         $server_status = "explicitly-started";
    131         last;
    132         }
    133     }
    134 
    135     if ($server_status eq "explicitly-started") {
    136         $self->{'jetty_explicitly_started'} = 1;
    137         print STDERR "Jetty server ready and listening for connections\n";
    138     }
    139     elsif ($server_status eq "already-running") {
    140         print STDERR "Using existing server detected on port $jetty_server_port\n";
    141     }
    142     else {
    143         print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n";
    144         exit -1;
    145     }
    146        
    147     # now we know the server is ready to accept connections, fork a
    148     # child process that continues to listen to the output and
    149     # prints out any lines that are not INFO lines
    150 
    151     if (fork()==0) {
    152         # child process
    153        
    154         my $line;
    155         while (defined ($line = <SIN>)) {
    156         next if ($line =~ m/^INFO:/);
    157         next if ($line =~ m/^[0-9 :-]*INFO::/);
    158         next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/);
    159         }
    160         close(SIN);
    161        
    162         # And now stop nicely
    163         exit 0;
    164     }
    165     }
    166     else {
    167     print STDERR "Error: failed to start solr-jetty-server\n";
    168     print STDERR "!$\n\n";
    169     print STDERR "Command attempted was:\n";
    170     print STDERR "  $server_java_cmd\n";
    171     print STDERR "run from directory:\n";
    172     print STDERR "  $solr_home\n";
    173     print STDERR "----\n";
    174 
    175     exit -1;
    176     }
    177 
    178     # If get to here then server started (and ready and listening)
    179     # *and* we are the parent process of the fork()
    180 
    181 }
    182 
    183 
    184 
    185 sub stop_solr_server
    186 {
    187     my $full_server_jar = $self->{'full_server_jar'};
    188     my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
    189    
    190     my $server_props = "-DSTOP.PORT=$jetty_stop_port";
    191     $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
    192     my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
    193 
    194     my $server_status = system($server_java_cmd);
    195    
    196     if ($server_status!=0) {
    197     print STDERR "Error: failed to stop solr-jetty-server\n";
    198     print STDERR "!$\n";
    199     exit -1;
    200     }
    201     else {
    202     wait(); # let the child process finish
    203     print STDERR "Jetty server shutdown\n";
    204     }
    205 }
    206 
     56my $self = { 'solr_server' => undef };
    20757
    20858sub open_java_solr
    20959{
    21060  my ($collect, $doc_tag_level,$full_builddir,$indexdir,$removeold) = @_;
    211 
    21261
    21362  # if removeold set, then delete the curring $full_builddir
     
    21766  }
    21867
    219   my $search_path = [];
    220 
    221   push(@$search_path,$ENV{'GSDLCOLLECTDIR'}) if defined $ENV{'GSDLCOLLECTDIR'};
    222   push(@$search_path,$ENV{'GSDLHOME'})       if defined $ENV{'GSDLHOME'};
    223   push(@$search_path,$ENV{'GEXT_SOLR'})      if defined $ENV{'GEXT_SOLR'};
    224 
    225 
    226   # The following returns once Jetty has generated its
    227   # "reading and listening" line
    228   #
    229   start_solr_server($search_path);
    230 
    231   # Now run the solr-post command
    232 
    233   chdir($ENV{'GEXT_SOLR'});
     68  # If the Solr/Jetty server is not already running, the following starts
     69  # it up, and only returns when the server is "reading and listening"
    23470 
    235   my $post_jar   = &util::filename_cat("lib","java","solr-post.jar");
    236   my $full_post_jar   = locate_file($search_path,$post_jar);
    237 
    238   my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
    239 
    240   # Now run solr-post command
    241   my $post_props = "-Durl=http://localhost:$jetty_server_port/solr/$collect-$doc_tag_level/update";
    242   $post_props .= " -Ddata=stdin";
    243   $post_props .= " -Dcommit=yes";
    244 
    245   my $post_java_cmd = "java $post_props -jar \"$full_post_jar\"";
    246 
    247 ###  print STDERR "**** post cmd = $post_java_cmd\n";
    248  
    249   open (PIPEOUT, "| $post_java_cmd")
    250       || die "Error in solr_passes.pl: Failed to run $post_java_cmd\n!$\n";
    251 }
    252 
    253 
     71  my $solr_server = new solrserver();
     72  $solr_server->start();
     73  $self->{'solr_server'} = $solr_server;
     74
     75  # Now start up the solr-post command
     76  &solrutil::open_post_pipe($collect,$doc_tag_level);
     77}
    25478
    25579sub close_java_solr
    25680{
    257     # closing the pipe has the effect of shutting down solr-post.jar
    258     close(PIPEOUT);
    259    
    260     if ($self->{'jetty_explicitly_started'}) {
    261     stop_solr_server();
    262     }
    263 }
    264 
     81    &solrutil::close_post_pipe();
     82     
     83    my $solr_server = $self->{'solr_server'};
     84    if ($solr_server->explicitly_started()) {
     85    $solr_server->stop();
     86    }
     87}
    26588
    26689#----
     
    348171    my $line;
    349172    while (defined ($line = <STDIN>)) {
    350     print PIPEOUT $line;
     173    &solrutil::print_to_post_pipe($line);
    351174    }
    352175}
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24447 r24453  
    3131
    3232use lucenebuilder;
     33use solrserver;
    3334use Config; # for getting the perlpath in the recommended way
    3435
     
    271272
    272273
     274sub solr_core_admin
     275{
     276    my $self = shift (@_);
     277    my ($url) = @_;
     278
     279    my $cmd = "wget -q -O - \"$url\"";
     280
     281    my $status = undef;
     282
     283    if (open(WIN,"$cmd |")) {
     284
     285    my $xml_output = "";
     286    my $line;
     287    while (defined ($line=<WIN>)) {
     288
     289        $xml_output .= $line;
     290    }
     291    close(WIN);
     292
     293##  print $xml_output;
     294
     295    ($status) = ($xml_output =~ m!<int name="status">(\d+)</int>!s);
     296   
     297    }
     298    else {
     299    print STDERR "Warning: failed to run $cmd\n";
     300    print STDERR "  $!\n";
     301    }
     302
     303    return $status;
     304
     305}
     306
    273307sub pre_build_indexes
    274308{
     
    277311    my $outhandle = $self->{'outhandle'};
    278312
    279     # read in build.cfg if in incremental mode???
     313    # If the Solr/Jetty server is not already running, the following starts
     314    # it up, and only returns when the server is "reading and listening"
     315 
     316    my $solr_server = new solrserver();
     317    $solr_server->start();
     318    $self->{'solr_server'} = $solr_server;
    280319
    281320    my $indexes = [];
     
    350389    }
    351390
    352     # write out solr 'schema.xml' (and related) file
     391    # Write out solr 'schema.xml' (and related) file
     392    #
    353393    $self->make_final_field_list();
    354394    $self->premake_solr_auxiliary_files();
    355395
    356     # if collect==core not already in solr.xml (check with STATUS)
    357     # => use CREATE API to add to solr.xml
    358     #
    359     # else
    360     # => use RELOAD call to refresh fields now expressed in schema.xml
     396    # Now update the solr-core information in solr.xml
     397    # => at most two cores <colname>-Doc and <colname>-Sec
     398
     399    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
     400    my $base_url = "http://localhost:$jetty_server_port/solr/admin/cores";
     401
     402    my $collection = $self->{'collection'};
     403
     404    foreach my $level (keys %{$self->{'levels'}}) {
     405
     406    my $llevel = $mgppbuilder::level_map{$level};
     407
     408    my $core = $collection."-".lc($llevel);
     409    my $check_core_url = "$base_url?action=STATUS&core=$core";
     410
     411    my $check_status = $self->solr_core_admin($check_core_url);
     412   
     413    print STDERR "*** check status = $check_status\n";
     414
     415
     416    # if collect==core not already in solr.xml (check with STATUS)
     417    # => use CREATE API to add to solr.xml
     418    #
     419    # else
     420    # => use RELOAD call to refresh fields now expressed in schema.xml
     421    }
    361422
    362423}
     
    499560    #  $self->make_final_field_list()
    500561    # as this has been done in our pre_build_indexes() phase for solr
     562
     563
     564    # Also need to stop the Solr/jetty server if it was explicitly started
     565    # in pre_build_indexes()
    501566   
     567    my $solr_server = $self->{'solr_server'};
     568
     569    if ($solr_server->explicitly_started()) {
     570    $solr_server->stop();
     571    }
     572
     573    $self->{'solr_server'} = undef;
     574
    502575}   
    503576
Note: See TracChangeset for help on using the changeset viewer.