Changeset 24453 for gs3-extensions/solr/trunk

Timestamp:

2011-08-23T14:23:40+12:00 (12 years ago)

Author:

davidb

Message:

Tidy up of code. Better structuring of classes

Location:

gs3-extensions/solr/trunk/src

Files:

• bin/script/solr_passes.pl (modified) (3 diffs)
• perllib/solrbuilder.pm (modified) (5 diffs)
• perllib/solrserver.pm (added)
• perllib/solrutil.pm (added)

gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

@@ -41 +41 @@
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
     die "GEXT_SOLR not set\n" unless defined $ENV{'GEXT_SOLR'};
+
+    my $solr_ext = $ENV{'GEXT_SOLR'};
+    unshift (@INC, "$solr_ext/perllib");
 }
 
 use strict;
 use util;
+use solrutil;
+use solrserver;
+
 
 # Not quite OO, but close enough for now
 #
-my $self = { 'full_server_jar' => undef,
-         'jetty_explicitly_started' => undef,
-         'jetty_stop_key' => "greenstone-solr"
-         };
-
-
-
-sub locate_file
-{
-    my ($search_path,$suffix) = @_;
-        
-    foreach my $sp (@$search_path) {
-    my $full_path = &util::filename_cat($sp,$suffix);
-    
-    if (-f $full_path) {
-        return $full_path;
-    }
-    }
-    
-    # if get to here, then failed to find match
-
-    print STDERR "Error: Failed to find '$suffix'\n";
-    print STDERR "  Looked in: ", join(", ", @$search_path), "\n";
-    exit -1;
-}
-
-sub start_solr_server
-{
-    my ($search_path) = @_;
-
-    my $solr_home         = $ENV{'GEXT_SOLR'};
-    my $jetty_stop_port   = $ENV{'JETTY_STOP_PORT'};
-    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
-
-    chdir($solr_home);
-    
-    my $solr_etc = &util::filename_cat($solr_home,"etc");
-
-    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
-    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
-    $server_props .= " -Dsolr.solr.home=$solr_etc";
-
-    my $server_jar = &util::filename_cat("lib","java","solr-jetty-server.jar");
-    my $full_server_jar = locate_file($search_path,$server_jar);
-    $self->{'full_server_jar'} = $full_server_jar;
-    
-    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
-
-##    print STDERR "**** server cmd = $server_java_cmd\n";
-
-    if (open(SIN,"$server_java_cmd 2>&1 |")) {
-    
-    my $server_status = "unknown";
-
-    my $line;
-    while (defined($line=<SIN>)) {
-        # Scan through output until you see a line like:
-        #   2011-08-22 .. :INFO::Started [email protected]:8983
-        # which signifies that the server has started up and is
-        # "ready and listening"
-
-##      print STDERR "**** $line";
-
-        if (($line =~ m/^(WARN|ERROR|SEVERE):/)
-        || ($line =~ m/^[0-9 :-]*(WARN|ERROR|SEVERE)::/)) {
-        print $line;
-        }
-
-
-        if ($line =~ m/WARN::failed SocketConnector/) {
-        if ($line =~ m/Address already in use/) {
-            $server_status = "already-running";
-        }
-        else {
-            $server_status = "failed-to-start";
-        }
-        last;
-        }
-        
-        if ($line =~ m/INFO::Started SocketConnector/) {
-        $server_status = "explicitly-started";
-        last;
-        }
-    }
-
-    if ($server_status eq "explicitly-started") {
-        $self->{'jetty_explicitly_started'} = 1;
-        print STDERR "Jetty server ready and listening for connections\n";
-    }
-    elsif ($server_status eq "already-running") {
-        print STDERR "Using existing server detected on port $jetty_server_port\n";
-    }
-    else {
-        print STDERR "Failed to start Solr/Jetty web server on $jetty_server_port\n";
-        exit -1;
-    }
-        
-    # now we know the server is ready to accept connections, fork a
-    # child process that continues to listen to the output and
-    # prints out any lines that are not INFO lines
-
-    if (fork()==0) {
-        # child process
-        
-        my $line;
-        while (defined ($line = <SIN>)) {
-        next if ($line =~ m/^INFO:/);
-        next if ($line =~ m/^[0-9 :-]*INFO::/);
-        next if ($line =~ m/^\d{2}\/\d{2}\/\d{4}\s+/); 
-        }
-        close(SIN);
-        
-        # And now stop nicely
-        exit 0;
-    }
-    }
-    else {
-    print STDERR "Error: failed to start solr-jetty-server\n";
-    print STDERR "!$\n\n";
-    print STDERR "Command attempted was:\n";
-    print STDERR "  $server_java_cmd\n";
-    print STDERR "run from directory:\n";
-    print STDERR "  $solr_home\n";
-    print STDERR "----\n";
-
-    exit -1;
-    }
-
-    # If get to here then server started (and ready and listening)
-    # *and* we are the parent process of the fork()
-
-}
-
-
-
-sub stop_solr_server
-{
-    my $full_server_jar = $self->{'full_server_jar'};
-    my $jetty_stop_port = $ENV{'JETTY_STOP_PORT'};
-    
-    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
-    $server_props   .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
-    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\" --stop";
-
-    my $server_status = system($server_java_cmd);
-    
-    if ($server_status!=0) {
-    print STDERR "Error: failed to stop solr-jetty-server\n";
-    print STDERR "!$\n";
-    exit -1;
-    }
-    else {
-    wait(); # let the child process finish
-    print STDERR "Jetty server shutdown\n";
-    }
-}
-
+my $self = { 'solr_server' => undef };
 
 sub open_java_solr
 {
   my ($collect, $doc_tag_level,$full_builddir,$indexdir,$removeold) = @_;
-
 
   # if removeold set, then delete the curring $full_builddir
@@ -217 +66 @@
   }
 
-  my $search_path = [];
-
-  push(@$search_path,$ENV{'GSDLCOLLECTDIR'}) if defined $ENV{'GSDLCOLLECTDIR'};
-  push(@$search_path,$ENV{'GSDLHOME'})       if defined $ENV{'GSDLHOME'};
-  push(@$search_path,$ENV{'GEXT_SOLR'})      if defined $ENV{'GEXT_SOLR'};
-
-
-  # The following returns once Jetty has generated its 
-  # "reading and listening" line
-  #
-  start_solr_server($search_path);
-
-  # Now run the solr-post command
-
-  chdir($ENV{'GEXT_SOLR'});
+  # If the Solr/Jetty server is not already running, the following starts
+  # it up, and only returns when the server is "reading and listening"
   
-  my $post_jar   = &util::filename_cat("lib","java","solr-post.jar");
-  my $full_post_jar   = locate_file($search_path,$post_jar);
-
-  my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
-
-  # Now run solr-post command
-  my $post_props = "-Durl=http://localhost:$jetty_server_port/solr/$collect-$doc_tag_level/update";
-  $post_props .= " -Ddata=stdin";
-  $post_props .= " -Dcommit=yes";
-
-  my $post_java_cmd = "java $post_props -jar \"$full_post_jar\"";
-
-###  print STDERR "**** post cmd = $post_java_cmd\n";
-  
-  open (PIPEOUT, "| $post_java_cmd") 
-      || die "Error in solr_passes.pl: Failed to run $post_java_cmd\n!$\n";
-}
-
-
+  my $solr_server = new solrserver();
+  $solr_server->start();
+  $self->{'solr_server'} = $solr_server;
+
+  # Now start up the solr-post command
+  &solrutil::open_post_pipe($collect,$doc_tag_level);
+}
 
 sub close_java_solr
 {
-    # closing the pipe has the effect of shutting down solr-post.jar
-    close(PIPEOUT);
-    
-    if ($self->{'jetty_explicitly_started'}) {
-    stop_solr_server();
-    }
-}
-
+    &solrutil::close_post_pipe();
+      
+    my $solr_server = $self->{'solr_server'};
+    if ($solr_server->explicitly_started()) {
+    $solr_server->stop();
+    }
+}
 
 #----
@@ -348 +171 @@
     my $line;
     while (defined ($line = <STDIN>)) {
-    print PIPEOUT $line;
+    &solrutil::print_to_post_pipe($line);
     }
 }

gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

@@ -31 +31 @@
 
 use lucenebuilder;
+use solrserver;
 use Config; # for getting the perlpath in the recommended way
 
@@ -271 +272 @@
 
 
+sub solr_core_admin
+{
+    my $self = shift (@_);
+    my ($url) = @_;
+
+    my $cmd = "wget -q -O - \"$url\"";
+
+    my $status = undef;
+
+    if (open(WIN,"$cmd |")) {
+
+    my $xml_output = "";
+    my $line;
+    while (defined ($line=<WIN>)) {
+
+        $xml_output .= $line;
+    }
+    close(WIN);
+
+##  print $xml_output;
+
+    ($status) = ($xml_output =~ m!<int name="status">(\d+)</int>!s);
+    
+    }
+    else {
+    print STDERR "Warning: failed to run $cmd\n";
+    print STDERR "  $!\n";
+    }
+
+    return $status;
+
+}
+
 sub pre_build_indexes
 {
@@ -277 +311 @@
     my $outhandle = $self->{'outhandle'};
 
-    # read in build.cfg if in incremental mode???
+    # If the Solr/Jetty server is not already running, the following starts
+    # it up, and only returns when the server is "reading and listening"
+  
+    my $solr_server = new solrserver();
+    $solr_server->start();
+    $self->{'solr_server'} = $solr_server;
 
     my $indexes = [];
@@ -350 +389 @@
     }
 
-    # write out solr 'schema.xml' (and related) file 
+    # Write out solr 'schema.xml' (and related) file 
+    #
     $self->make_final_field_list();
     $self->premake_solr_auxiliary_files();
 
-    # if collect==core not already in solr.xml (check with STATUS)
-    # => use CREATE API to add to solr.xml
-    #
-    # else 
-    # => use RELOAD call to refresh fields now expressed in schema.xml
+    # Now update the solr-core information in solr.xml
+    # => at most two cores <colname>-Doc and <colname>-Sec
+
+    my $jetty_server_port = $ENV{'SOLR_JETTY_PORT'};
+    my $base_url = "http://localhost:$jetty_server_port/solr/admin/cores";
+
+    my $collection = $self->{'collection'};
+
+    foreach my $level (keys %{$self->{'levels'}}) {
+
+    my $llevel = $mgppbuilder::level_map{$level};
+
+    my $core = $collection."-".lc($llevel);
+    my $check_core_url = "$base_url?action=STATUS&core=$core";
+
+    my $check_status = $self->solr_core_admin($check_core_url);
+    
+    print STDERR "*** check status = $check_status\n";
+
+
+    # if collect==core not already in solr.xml (check with STATUS)
+    # => use CREATE API to add to solr.xml
+    #
+    # else 
+    # => use RELOAD call to refresh fields now expressed in schema.xml
+    }
 
 }
@@ -499 +560 @@
     #  $self->make_final_field_list()
     # as this has been done in our pre_build_indexes() phase for solr
+
+
+    # Also need to stop the Solr/jetty server if it was explicitly started
+    # in pre_build_indexes()
     
+    my $solr_server = $self->{'solr_server'};
+
+    if ($solr_server->explicitly_started()) {
+    $solr_server->stop();
+    }
+
+    $self->{'solr_server'} = undef;
+
 }