Changeset 24447 for gs3-extensions

Show
Ignore:
Timestamp:
22.08.2011 14:17:35 (8 years ago)
Author:
davidb
Message:

Tidy up of code (removing commented out redundant code), plus tweaking of code that starts and stops jetty to cope with situation where the server is already running

Location:
gs3-extensions/solr/trunk/src
Files:
3 modified

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24446 r24447  
    3535# into that needed by the solr server.   
    3636 
    37 my $jetty_stop_key="greenstone-solr"; 
    3837 
    3938BEGIN { 
     
    4746use util; 
    4847 
    49  
    5048# Not quite OO, but close enough for now 
    5149# 
    5250my $self = { 'full_server_jar' => undef, 
    53 ##       'full_post_jar'   = undef, 
    54 ##       'full_index_propfile' = undef, 
    55          'jetty_explicitly_started' => undef 
    56      }; 
     51         'jetty_explicitly_started' => undef, 
     52         'jetty_stop_key' => "greenstone-solr" 
     53         }; 
    5754 
    5855 
     
    8178    my ($search_path) = @_; 
    8279 
    83     my $solr_home       = $ENV{'GEXT_SOLR'}; 
    84     my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'}; 
     80    my $solr_home         = $ENV{'GEXT_SOLR'}; 
     81    my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'}; 
     82    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
    8583 
    8684    chdir($solr_home); 
     
    8987 
    9088    my $server_props = "-DSTOP.PORT=$jetty_stop_port"; 
    91     $server_props .= " -DSTOP.KEY=$jetty_stop_key"; 
     89    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    9290    $server_props .= " -Dsolr.solr.home=$solr_etc"; 
    9391 
     
    9795     
    9896    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\""; 
    99 #    if ($ENV{'GSDLOS'} eq "windows") { 
    100 #   $server_java_cmd = "start $server_java_cmd"; 
    101 #    } 
    102 #    else { 
    103 #   $server_java_cmd .= " &"; 
    104 #    } 
    105      
     97 
     98##    print STDERR "**** server cmd = $server_java_cmd\n"; 
    10699 
    107100    if (open(SIN,"$server_java_cmd 2>&1 |")) { 
    108101     
     102    my $server_status = "unknown"; 
     103 
    109104    my $line; 
    110105    while (defined($line=<SIN>)) { 
     
    113108        # which signifies that the server has started up and is 
    114109        # "ready and listening" 
     110 
     111##      print STDERR "**** $line"; 
     112 
     113        if (($line =~ m/^(WARN|ERROR|SEVERE):/) 
     114        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) { 
     115        print $line; 
     116        } 
     117 
     118 
     119        if ($line =~ m/WARN::failed SocketConnector/) { 
     120        if ($line =~ m/Address already in use/) { 
     121            $server_status = "already-running"; 
     122        } 
     123        else { 
     124            $server_status = "failed-to-start"; 
     125        } 
     126        last; 
     127        } 
     128         
     129        if ($line =~ m/INFO::Started SocketConnector/) { 
     130        $server_status = "explicitly-started"; 
     131        last; 
     132        } 
     133    } 
     134 
     135    if ($server_status eq "explicitly-started") { 
     136        $self->{'jetty_explicitly_started'} = 1; 
     137        print STDERR "Jetty server ready and listening for connections\n"; 
     138    } 
     139    elsif ($server_status eq "already-running") { 
     140        print STDERR "Using existing server detected on port $jetty_server_port\n"; 
     141    } 
     142    else { 
     143        print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n"; 
     144        exit -1; 
     145    } 
    115146         
    116         last if ($line =~ m/INFO::Started SocketConnector/); 
    117     } 
    118  
    119     print STDERR "Jetty server ready and listening for connections\n"; 
    120  
    121147    # now we know the server is ready to accept connections, fork a 
    122148    # child process that continues to listen to the output and 
     
    140166    else { 
    141167    print STDERR "Error: failed to start solr-jetty-server\n"; 
    142     print STDERR "!$\n"; 
     168    print STDERR "!$\n\n"; 
     169    print STDERR "Command attempted was:\n"; 
     170    print STDERR "  $server_java_cmd\n"; 
     171    print STDERR "run from directory:\n"; 
     172    print STDERR "  $solr_home\n"; 
     173    print STDERR "----\n"; 
     174 
    143175    exit -1; 
    144176    } 
     
    147179    # *and* we are the parent process of the fork() 
    148180 
    149     $self->{'jetty_explicitly_started'} = 1; 
    150  
    151181} 
    152182 
     
    158188    my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'}; 
    159189     
    160     my $server_props = "-DSTOP.PORT=$jetty_stop_port -DSTOP.KEY=$jetty_stop_key"; 
     190    my $server_props = "-DSTOP.PORT=$jetty_stop_port"; 
     191    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    161192    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop"; 
    162193 
     
    170201    else { 
    171202    wait(); # let the child process finish 
     203    print STDERR "Jetty server shutdown\n"; 
    172204    } 
    173205} 
     
    185217  } 
    186218 
    187 # No longer used, as solr now run as two cores per collection (-Doc and -Sec) 
    188 # 
    189 #  # Set up an 'index.properties' file in $full_builddir, with the line 
    190 #  #   index=$indexdir 
    191 #  $full_index_propfile = &util::filename_cat($full_builddir,"index.properties"); 
    192  
    193 #  if (open(IPOUT,">$full_index_propfile")) { 
    194 #      print IPOUT "index=$indexdir\n"; 
    195 #      close(IPOUT); 
    196 #  } 
    197 #  else { 
    198 #      print STDERR "Failed to create $full_index_propfile\n"; 
    199 #      print STDERR "!$\n"; 
    200 #      exit -2; 
    201 #  } 
    202  
    203    
    204  
    205219  my $search_path = []; 
    206220 
     
    215229  start_solr_server($search_path); 
    216230 
    217  
    218231  # Now run the solr-post command 
    219232 
    220233  chdir($ENV{'GEXT_SOLR'}); 
    221  
    222234   
    223235  my $post_jar   = &util::filename_cat("lib","java","solr-post.jar"); 
    224236  my $full_post_jar   = locate_file($search_path,$post_jar); 
    225 ##  $self->{'full_post_jar'} = $full_post_jar; 
    226237 
    227238  my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'}; 
     
    234245  my $post_java_cmd = "java $post_props -jar \"$full_post_jar\""; 
    235246 
    236   print STDERR "**** post cmd = $post_java_cmd\n"; 
     247###  print STDERR "**** post cmd = $post_java_cmd\n"; 
    237248   
    238249  open (PIPEOUT, "| $post_java_cmd")  
     
    250261    stop_solr_server(); 
    251262    } 
    252  
    253 #  No longer used 
    254 #    # $full_index_propfile is set up as a global variable so it can be shared 
    255 #    # between open_java_solr() and here 
    256 #    &util::rm($full_index_propfile); 
    257 } 
    258  
    259  
    260  
    261  
     263} 
     264 
     265 
     266#---- 
    262267 
    263268sub save_xml_doc 
     
    418423  } 
    419424  else { 
    420       print STDERR "Streaming input onto solr server!\n"; 
     425      print STDERR "Streaming document input onto Solr server!\n"; 
    421426      pass_on_xml_stream(); 
    422427  } 
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24446 r24447  
    320320    # to figure out what the metadata_field_mapping is       
    321321 
    322 ##  my $db_level = "section"; #always 
    323  
    324322    # set up the document processr 
    325323    $self->{'buildproc'}->set_output_handle (undef); 
    326324    $self->{'buildproc'}->set_mode ('index_field_mapping'); 
    327 ##  $self->{'buildproc'}->set_index ($index); 
    328 ##  $self->{'buildproc'}->set_indexing_text (0); 
    329     #$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'}); 
    330 ##  $self->{'buildproc'}->set_levels ($levels); 
    331 ##  $self->{'buildproc'}->set_db_level ($db_level); 
    332325    $self->{'buildproc'}->reset(); 
    333326     
     
    338331    &plugin::end($self->{'pluginfo'}); 
    339332     
    340 ##  close ($handle) unless $self->{'debug'}; 
    341  
    342 ##  $self->print_stats(); 
    343  
    344     # just make "delete" stop  ??? 
    345333    } 
    346334 
     
    373361 
    374362} 
    375  
    376 # Essentially the same as the lucenebuilder.pm version, only using solr_passes 
    377 # => refactor and make better use of inheritence 
    378  
    379 sub build_indexesXXXX { 
    380     my $self = shift (@_); 
    381     my ($indexname) = @_; 
    382     my $outhandle = $self->{'outhandle'}; 
    383  
    384     $self->pre_build_indexes($indexname); 
    385  
    386     my $indexes = []; 
    387     if (defined $indexname && $indexname =~ /\w/) { 
    388     push @$indexes, $indexname; 
    389     } else { 
    390     $indexes = $self->{'collect_cfg'}->{'indexes'}; 
    391     } 
    392  
    393     # have we got para index? 
    394     foreach my $level (keys %{$self->{'levels'}}) { 
    395     if ($level =~ /paragraph/) { 
    396         print $outhandle "Warning: Paragraph level indexing not supported by Solr\n"; 
    397         last; 
    398     } 
    399     } 
    400     # create the mapping between the index descriptions 
    401     # and their directory names (includes subcolls and langs) 
    402     $self->{'index_mapping'} = $self->create_index_mapping ($indexes); 
    403  
    404     # build each of the indexes 
    405     foreach my $index (@$indexes) { 
    406     if ($self->want_built($index)) { 
    407  
    408         my $idx = $self->{'index_mapping'}->{$index}; 
    409         foreach my $level (keys %{$self->{'levels'}}) { 
    410         next if $level =~ /paragraph/; # we don't do para indexing 
    411         my ($pindex) = $level =~ /^(.)/; 
    412         # should probably check that new name with level 
    413         # is unique ... but currently (with doc sec and para) 
    414         # each has unique first letter. 
    415         $self->{'index_mapping'}->{$index} = $pindex.$idx; 
    416  
    417         my $llevel = $mgppbuilder::level_map{$level}; 
    418         print $outhandle "\n*** building index $index at level $llevel in subdirectory " . 
    419             "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1); 
    420         print STDERR "<Stage name='Index' source='$index' level=$llevel>\n" if $self->{'gli'}; 
    421  
    422         $self->build_index($index,$llevel); 
    423         } 
    424         $self->{'index_mapping'}->{$index} = $idx; 
    425  
    426     } else { 
    427         print $outhandle "\n*** ignoring index $index\n" if ($self->{'verbosity'} >= 1); 
    428     } 
    429     } 
    430 } 
    431  
    432363 
    433364# Essentially the same as the lucenebuilder.pm version, only using solr_passes 
     
    546477    print $handle "<update>\n"; 
    547478 
    548     open(TOUT,">/tmp/solr.out"); binmode(TOUT,":utf8"); 
    549     print TOUT "<update>\n"; 
    550     close(TOUT); 
    551  
    552479    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
    553480           "", {}, {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'}); 
     
    556483    print $handle "</update>\n"; 
    557484 
    558     open(TOUT,">>/tmp/solr.out"); binmode(TOUT,":utf8"); 
    559     print TOUT "</update>\n"; 
    560     close(TOUT); 
    561  
    562485    close ($handle) unless $self->{'debug'}; 
    563486 
     
    568491 
    569492} 
    570  
    571493 
    572494 
  • gs3-extensions/solr/trunk/src/perllib/solrbuildproc.pm

    r24446 r24447  
    556556    $text .= $end_doc if ($sec_tag_name eq ""); 
    557557 
    558  
    559558##    $text .= "<commit/>\n"; 
    560559 
    561560    print $solrhandle $text; 
    562561 
    563     open(TOUT,">>/tmp/solr.out"); binmode(TOUT,":utf8"); 
    564     print TOUT $text; 
    565     close(TOUT); 
    566562} 
    567563