Changeset 24501 for gs3-extensions

Timestamp:

2011-08-26T23:32:25+12:00 (13 years ago)

Author:

davidb

Message:

Relocation of files to make solr.solr.home more natural. Plus, more carefully control the order in which the build_dir/index_dir folder is deleted in. For solr we need to do this earlier than lucene

Location:

gs3-extensions/solr/trunk/src

Files:

• bin/script/solr_passes.pl (modified) (4 diffs)
• perllib/solrbuilder.pm (modified) (6 diffs)
• perllib/solrserver.pm (modified) (6 diffs)
• perllib/solrutil.pm (modified) (3 diffs)

gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

@@ -58 +58 @@
 sub open_java_solr
 {
-  my ($collect, $ds_idx,$full_builddir,$indexdir,$removeold) = @_;
-
-  # if removeold set, then delete the curring $full_builddir
-  if ($removeold) {
-      my $full_indexdir = &util::filename_cat($full_builddir,$indexdir);
-      &util::rm_r($full_indexdir);
-  }
+  my ($core,$full_builddir,$indexdir) = @_;
 
   # If the Solr/Jetty server is not already running, the following starts
   # it up, and only returns when the server is "reading and listening"
   
-  my $solr_server = new solrserver();
+  my $solr_server = new solrserver($full_builddir);
   $solr_server->start();
   $self->{'solr_server'} = $solr_server;
 
   # Now start up the solr-post command
-  &solrutil::open_post_pipe($collect,$ds_idx);
+  &solrutil::open_post_pipe($core);
 }
 
@@ -187 +181 @@
   my $argc = scalar(@argv);
 
-  my $removeold = 0;
   my @filtered_argv = ();
 
@@ -196 +189 @@
       my $option = $1;
 
-      # -removeold causes the existing index to be overwritten
-      if ($option eq "removeold") {
-        print STDERR "\n-removeold set (new index will be created)\n";
-        $removeold = 1;
-      }
       # -verbosity <num>
-      elsif ($option eq "verbosity") {
+      if ($option eq "verbosity") {
         $i++;
         if ($i<$argc)
@@ -223 +211 @@
   my $filtered_argc = scalar(@filtered_argv);
 
-  if ($filtered_argc < 5) {
-    print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" {d|s}idx build-dir index-name\n";
+  if ($filtered_argc < 4) {
+    print STDERR "Usage: solr_passes.pl [-verbosity num] core \"text\"|\"index\" build-dir index-name\n";
     exit 1;
   }
 
-  my $collect       = $filtered_argv[0];
+  my $core          = $filtered_argv[0];
   my $mode          = $filtered_argv[1];
-  my $ds_idx        = $filtered_argv[2];
-  my $full_builddir = $filtered_argv[3];
-  my $indexdir      = $filtered_argv[4];
+  my $full_builddir = $filtered_argv[2];
+  my $indexdir      = $filtered_argv[3];
 
   # We only need the Solr handle opened if we are indexing the
   # documents, not if we are just storing the text
   if ($mode eq "index") {
-    open_java_solr($collect, $ds_idx, $full_builddir, $indexdir, $removeold);
+    open_java_solr($core, $full_builddir, $indexdir);
   }
 

gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

@@ -112 +112 @@
     else
     {
-    my $collection = $self->{'collection'};
+    my $site        = $self->{'site'};
+    my $collect     = $self->{'collection'};    
+    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
+    my $core        = $core_prefix; # unused in this call to solr_passes
 
         print STDERR "Executable:    $solr_passes_exe\n";
         print STDERR "Sections:      $solr_passes_sections\n";
         print STDERR "Build Dir:     $build_dir\n";
-        print STDERR "Cmd:           $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra\n";
-    if (!open($handle, "| $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra"))
+        print STDERR "Cmd:           $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra\n";
+    if (!open($handle, "| $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra"))
     {
         print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
@@ -243 +246 @@
         
     my $solr_home = $ENV{'GEXT_SOLR'};
-    my $in_dirname = &util::filename_cat($solr_home,"etc","conf");
+##    my $in_dirname = &util::filename_cat($solr_home,"etc","conf");
+    my $in_dirname = &util::filename_cat($solr_home,"conf");
     my $schema_in_filename = &util::filename_cat($in_dirname,"schema.xml.in");
 
@@ -281 +285 @@
     # it up, and only returns when the server is "reading and listening"
   
-    my $solr_server = new solrserver();
+    my $solr_server = new solrserver($self->{'build_dir'});
     $solr_server->start();
     $self->{'solr_server'} = $solr_server;
@@ -364 +368 @@
     # => at most two cores <colname>-Doc and <colname>-Sec
 
-    my $collection = $self->{'collection'};
+    my $site        = $self->{'site'};
+    my $collect     = $self->{'collection'};
+    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
 
     # my $idx = $self->{'index_mapping'}->{$index};
     my $idx = "idx";
 
-    my $site = $self->{'site'};
+    my $build_dir = $self->{'build_dir'};
 
     foreach my $level (keys %{$self->{'levels'}}) {
     
     my ($pindex) = $level =~ /^(.)/;
-    
-    my $core = $collection."-".$pindex.$idx;
-
-    # prefix site if exists (e.g. Greenstone 3)
-    $core = "$site-$core" if defined $site;
+
+    my $index_dir = $pindex.$idx;
+    my $core = "$core_prefix-$index_dir";
+
+    my $force_removeold = ($self->{'incremental'}) ? 0 : 1;
+    if ($force_removeold) {
+        print $outhandle "\n-removeold set (new index will be created)\n";
+
+        my $full_index_dir = &util::filename_cat($build_dir,$index_dir);
+        &util::rm_r($full_index_dir);
+        &util::mk_dir($full_index_dir);
+    }
 
     # if collect==core already in solr.xml (check with STATUS)
@@ -421 +434 @@
     my $solr_passes_sections = $llevel;
 
-    my $opt_create_index = ($self->{'incremental'}) ? "" : "-removeold";
-
     my $osextra = "";
     if ($ENV{'GSDLOS'} =~ /^windows$/i) {
@@ -478 +489 @@
     $handle = *STDOUT;
     } else {
-    my $collection = $self->{'collection'};
-    my $ds_idx = $self->{'index_mapping'}->{$index};
-
-    print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra\n";
-    if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra")) {
+    my $site        = $self->{'site'};
+    my $collect     = $self->{'collection'};
+    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
+    my $ds_idx      = $self->{'index_mapping'}->{$index};
+    my $core        = "$core_prefix-$ds_idx";
+
+    print STDERR "Cmd: $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra\n";
+    if (!open($handle, "| $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra")) {
         print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
         die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n";

gs3-extensions/solr/trunk/src/perllib/solrserver.pm

@@ -34 +34 @@
 sub new {
     my $class = shift(@_);
+    my ($build_dir) = @_;
 
     my $self = { 'jetty_stop_key' => "greenstone-solr" };
+
+    $self->{'build_dir'} = $build_dir;
 
     my $search_path = &solrutil::get_search_path();
@@ -74 +77 @@
     my $in_preamble = ($output_format eq "xml") ? 1 : 0;
     
+##    print STDERR "**** wgetcmd = \n $cmd\n";
+
     if (open(WIN,"$cmd |")) {
 
@@ -213 +218 @@
 {
     my $self = shift @_;
-    my ($core) = @_;
-
-    my ($ds_idx) = ($core =~ m/^.*-(.*)$/);
+    my ($core,$removeold) = @_;
+
+    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
 
     my $cgi_get_args = "action=CREATE&name=$core";
@@ -243 +248 @@
     chdir($solr_home);
     
-    my $solr_etc = &util::filename_cat($solr_home,"etc");
+##    my $solr_etc = &util::filename_cat($solr_home,"etc");
 
     my $server_props = "-DSTOP.PORT=$jetty_stop_port";
     $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
-    $server_props .= " -Dsolr.solr.home=$solr_etc";
+    $server_props .= " -Dsolr.solr.home=$solr_home";
 
     my $full_server_jar = $self->{'full_server_jar'};
@@ -253 +258 @@
     my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
 
-##    print STDERR "**** server cmd start = $server_java_cmd\n";
 
     my $server_status = "unknown";
@@ -261 +265 @@
     }
     elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
+
+##  print STDERR "**** startup up server with cmd start =\n $server_java_cmd\n";
 
     my $line;

gs3-extensions/solr/trunk/src/perllib/solrutil.pm

@@ -63 +63 @@
 sub open_post_pipe
 {
-    my ($collect,$ds_idx) = @_;
+    my ($core) = @_;
 
     my $search_path = get_search_path();
@@ -75 +75 @@
     
     # Now run solr-post command
-    my $core = $collect."-".$ds_idx;
     my $post_props = "-Durl=http://localhost:$jetty_port/solr/$core/update";
     $post_props .= " -Ddata=stdin";
@@ -82 +81 @@
     my $post_java_cmd = "java $post_props -jar \"$full_post_jar\"";
     
-###  print STDERR "**** post cmd = $post_java_cmd\n";
+##  print STDERR "**** post cmd = $post_java_cmd\n";
     
     open (PIPEOUT, "| $post_java_cmd")