Changeset 24501

Show
Ignore:
Timestamp:
26.08.2011 23:32:25 (8 years ago)
Author:
davidb
Message:

Relocation of files to make solr.solr.home more natural. Plus, more carefully control the order in which the build_dir/index_dir folder is deleted in. For solr we need to do this earlier than lucene

Location:
gs3-extensions/solr/trunk/src
Files:
4 modified

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24483 r24501  
    5858sub open_java_solr 
    5959{ 
    60   my ($collect, $ds_idx,$full_builddir,$indexdir,$removeold) = @_; 
    61  
    62   # if removeold set, then delete the curring $full_builddir 
    63   if ($removeold) { 
    64       my $full_indexdir = &util::filename_cat($full_builddir,$indexdir); 
    65       &util::rm_r($full_indexdir); 
    66   } 
     60  my ($core,$full_builddir,$indexdir) = @_; 
    6761 
    6862  # If the Solr/Jetty server is not already running, the following starts 
    6963  # it up, and only returns when the server is "reading and listening" 
    7064   
    71   my $solr_server = new solrserver(); 
     65  my $solr_server = new solrserver($full_builddir); 
    7266  $solr_server->start(); 
    7367  $self->{'solr_server'} = $solr_server; 
    7468 
    7569  # Now start up the solr-post command 
    76   &solrutil::open_post_pipe($collect,$ds_idx); 
     70  &solrutil::open_post_pipe($core); 
    7771} 
    7872 
     
    187181  my $argc = scalar(@argv); 
    188182 
    189   my $removeold = 0; 
    190183  my @filtered_argv = (); 
    191184 
     
    196189      my $option = $1; 
    197190 
    198       # -removeold causes the existing index to be overwritten 
    199       if ($option eq "removeold") { 
    200         print STDERR "\n-removeold set (new index will be created)\n"; 
    201         $removeold = 1; 
    202       } 
    203191      # -verbosity <num> 
    204       elsif ($option eq "verbosity") { 
     192      if ($option eq "verbosity") { 
    205193        $i++; 
    206194        if ($i<$argc) 
     
    223211  my $filtered_argc = scalar(@filtered_argv); 
    224212 
    225   if ($filtered_argc < 5) { 
    226     print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" {d|s}idx build-dir index-name\n"; 
     213  if ($filtered_argc < 4) { 
     214    print STDERR "Usage: solr_passes.pl [-verbosity num] core \"text\"|\"index\" build-dir index-name\n"; 
    227215    exit 1; 
    228216  } 
    229217 
    230   my $collect       = $filtered_argv[0]; 
     218  my $core          = $filtered_argv[0]; 
    231219  my $mode          = $filtered_argv[1]; 
    232   my $ds_idx        = $filtered_argv[2]; 
    233   my $full_builddir = $filtered_argv[3]; 
    234   my $indexdir      = $filtered_argv[4]; 
     220  my $full_builddir = $filtered_argv[2]; 
     221  my $indexdir      = $filtered_argv[3]; 
    235222 
    236223  # We only need the Solr handle opened if we are indexing the 
    237224  # documents, not if we are just storing the text 
    238225  if ($mode eq "index") { 
    239     open_java_solr($collect, $ds_idx, $full_builddir, $indexdir, $removeold); 
     226    open_java_solr($core, $full_builddir, $indexdir); 
    240227  } 
    241228 
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24497 r24501  
    112112    else 
    113113    { 
    114     my $collection = $self->{'collection'}; 
     114    my $site        = $self->{'site'}; 
     115    my $collect     = $self->{'collection'};     
     116    my $core_prefix = (defined $site) ? "$site-$collect" : $collect; 
     117    my $core        = $core_prefix; # unused in this call to solr_passes 
    115118 
    116119        print STDERR "Executable:    $solr_passes_exe\n"; 
    117120        print STDERR "Sections:      $solr_passes_sections\n"; 
    118121        print STDERR "Build Dir:     $build_dir\n"; 
    119         print STDERR "Cmd:           $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra\n"; 
    120     if (!open($handle, "| $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra")) 
     122        print STDERR "Cmd:           $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra\n"; 
     123    if (!open($handle, "| $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra")) 
    121124    { 
    122125        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'}; 
     
    243246         
    244247    my $solr_home = $ENV{'GEXT_SOLR'}; 
    245     my $in_dirname = &util::filename_cat($solr_home,"etc","conf"); 
     248##    my $in_dirname = &util::filename_cat($solr_home,"etc","conf"); 
     249    my $in_dirname = &util::filename_cat($solr_home,"conf"); 
    246250    my $schema_in_filename = &util::filename_cat($in_dirname,"schema.xml.in"); 
    247251 
     
    281285    # it up, and only returns when the server is "reading and listening" 
    282286   
    283     my $solr_server = new solrserver(); 
     287    my $solr_server = new solrserver($self->{'build_dir'}); 
    284288    $solr_server->start(); 
    285289    $self->{'solr_server'} = $solr_server; 
     
    364368    # => at most two cores <colname>-Doc and <colname>-Sec 
    365369 
    366     my $collection = $self->{'collection'}; 
     370    my $site        = $self->{'site'}; 
     371    my $collect     = $self->{'collection'}; 
     372    my $core_prefix = (defined $site) ? "$site-$collect" : $collect; 
    367373 
    368374    # my $idx = $self->{'index_mapping'}->{$index}; 
    369375    my $idx = "idx"; 
    370376 
    371     my $site = $self->{'site'}; 
     377    my $build_dir = $self->{'build_dir'}; 
    372378 
    373379    foreach my $level (keys %{$self->{'levels'}}) { 
    374380     
    375381    my ($pindex) = $level =~ /^(.)/; 
    376      
    377     my $core = $collection."-".$pindex.$idx; 
    378  
    379     # prefix site if exists (e.g. Greenstone 3) 
    380     $core = "$site-$core" if defined $site; 
     382 
     383    my $index_dir = $pindex.$idx; 
     384    my $core = "$core_prefix-$index_dir"; 
     385 
     386    my $force_removeold = ($self->{'incremental'}) ? 0 : 1; 
     387    if ($force_removeold) { 
     388        print $outhandle "\n-removeold set (new index will be created)\n"; 
     389 
     390        my $full_index_dir = &util::filename_cat($build_dir,$index_dir); 
     391        &util::rm_r($full_index_dir); 
     392        &util::mk_dir($full_index_dir); 
     393    } 
    381394 
    382395    # if collect==core already in solr.xml (check with STATUS) 
     
    421434    my $solr_passes_sections = $llevel; 
    422435 
    423     my $opt_create_index = ($self->{'incremental'}) ? "" : "-removeold"; 
    424  
    425436    my $osextra = ""; 
    426437    if ($ENV{'GSDLOS'} =~ /^windows$/i) { 
     
    478489    $handle = *STDOUT; 
    479490    } else { 
    480     my $collection = $self->{'collection'}; 
    481     my $ds_idx = $self->{'index_mapping'}->{$index}; 
    482  
    483     print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra\n"; 
    484     if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra")) { 
     491    my $site        = $self->{'site'}; 
     492    my $collect     = $self->{'collection'}; 
     493    my $core_prefix = (defined $site) ? "$site-$collect" : $collect; 
     494    my $ds_idx      = $self->{'index_mapping'}->{$index}; 
     495    my $core        = "$core_prefix-$ds_idx"; 
     496 
     497    print STDERR "Cmd: $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra\n"; 
     498    if (!open($handle, "| $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra")) { 
    485499        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    486500        die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n"; 
  • gs3-extensions/solr/trunk/src/perllib/solrserver.pm

    r24486 r24501  
    3434sub new { 
    3535    my $class = shift(@_); 
     36    my ($build_dir) = @_; 
    3637 
    3738    my $self = { 'jetty_stop_key' => "greenstone-solr" }; 
     39 
     40    $self->{'build_dir'} = $build_dir; 
    3841 
    3942    my $search_path = &solrutil::get_search_path(); 
     
    7477    my $in_preamble = ($output_format eq "xml") ? 1 : 0; 
    7578     
     79##    print STDERR "**** wgetcmd = \n $cmd\n"; 
     80 
    7681    if (open(WIN,"$cmd |")) { 
    7782 
     
    213218{ 
    214219    my $self = shift @_; 
    215     my ($core) = @_; 
    216  
    217     my ($ds_idx) = ($core =~ m/^.*-(.*)$/); 
     220    my ($core,$removeold) = @_; 
     221 
     222    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/); 
    218223 
    219224    my $cgi_get_args = "action=CREATE&name=$core"; 
     
    243248    chdir($solr_home); 
    244249     
    245     my $solr_etc = &util::filename_cat($solr_home,"etc"); 
     250##    my $solr_etc = &util::filename_cat($solr_home,"etc"); 
    246251 
    247252    my $server_props = "-DSTOP.PORT=$jetty_stop_port"; 
    248253    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'}; 
    249     $server_props .= " -Dsolr.solr.home=$solr_etc"; 
     254    $server_props .= " -Dsolr.solr.home=$solr_home"; 
    250255 
    251256    my $full_server_jar = $self->{'full_server_jar'}; 
     
    253258    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\""; 
    254259 
    255 ##    print STDERR "**** server cmd start = $server_java_cmd\n"; 
    256260 
    257261    my $server_status = "unknown"; 
     
    261265    } 
    262266    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) { 
     267 
     268##  print STDERR "**** startup up server with cmd start =\n $server_java_cmd\n"; 
    263269 
    264270    my $line; 
  • gs3-extensions/solr/trunk/src/perllib/solrutil.pm

    r24483 r24501  
    6363sub open_post_pipe 
    6464{ 
    65     my ($collect,$ds_idx) = @_; 
     65    my ($core) = @_; 
    6666 
    6767    my $search_path = get_search_path(); 
     
    7575     
    7676    # Now run solr-post command 
    77     my $core = $collect."-".$ds_idx; 
    7877    my $post_props = "-Durl=http://localhost:$jetty_port/solr/$core/update"; 
    7978    $post_props .= " -Ddata=stdin"; 
     
    8281    my $post_java_cmd = "java $post_props -jar \"$full_post_jar\""; 
    8382     
    84 ###  print STDERR "**** post cmd = $post_java_cmd\n"; 
     83##  print STDERR "**** post cmd = $post_java_cmd\n"; 
    8584     
    8685    open (PIPEOUT, "| $post_java_cmd")