Changeset 24501


Ignore:
Timestamp:
2011-08-26T23:32:25+12:00 (11 years ago)
Author:
davidb
Message:

Relocation of files to make solr.solr.home more natural. Plus, more carefully control the order in which the build_dir/index_dir folder is deleted in. For solr we need to do this earlier than lucene

Location:
gs3-extensions/solr/trunk/src
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • gs3-extensions/solr/trunk/src/bin/script/solr_passes.pl

    r24483 r24501  
    5858sub open_java_solr
    5959{
    60   my ($collect, $ds_idx,$full_builddir,$indexdir,$removeold) = @_;
    61 
    62   # if removeold set, then delete the curring $full_builddir
    63   if ($removeold) {
    64       my $full_indexdir = &util::filename_cat($full_builddir,$indexdir);
    65       &util::rm_r($full_indexdir);
    66   }
     60  my ($core,$full_builddir,$indexdir) = @_;
    6761
    6862  # If the Solr/Jetty server is not already running, the following starts
    6963  # it up, and only returns when the server is "reading and listening"
    7064 
    71   my $solr_server = new solrserver();
     65  my $solr_server = new solrserver($full_builddir);
    7266  $solr_server->start();
    7367  $self->{'solr_server'} = $solr_server;
    7468
    7569  # Now start up the solr-post command
    76   &solrutil::open_post_pipe($collect,$ds_idx);
     70  &solrutil::open_post_pipe($core);
    7771}
    7872
     
    187181  my $argc = scalar(@argv);
    188182
    189   my $removeold = 0;
    190183  my @filtered_argv = ();
    191184
     
    196189      my $option = $1;
    197190
    198       # -removeold causes the existing index to be overwritten
    199       if ($option eq "removeold") {
    200         print STDERR "\n-removeold set (new index will be created)\n";
    201         $removeold = 1;
    202       }
    203191      # -verbosity <num>
    204       elsif ($option eq "verbosity") {
     192      if ($option eq "verbosity") {
    205193        $i++;
    206194        if ($i<$argc)
     
    223211  my $filtered_argc = scalar(@filtered_argv);
    224212
    225   if ($filtered_argc < 5) {
    226     print STDERR "Usage: solr_passes.pl [-removeold|-verbosity num] collect \"text\"|\"index\" {d|s}idx build-dir index-name\n";
     213  if ($filtered_argc < 4) {
     214    print STDERR "Usage: solr_passes.pl [-verbosity num] core \"text\"|\"index\" build-dir index-name\n";
    227215    exit 1;
    228216  }
    229217
    230   my $collect       = $filtered_argv[0];
     218  my $core          = $filtered_argv[0];
    231219  my $mode          = $filtered_argv[1];
    232   my $ds_idx        = $filtered_argv[2];
    233   my $full_builddir = $filtered_argv[3];
    234   my $indexdir      = $filtered_argv[4];
     220  my $full_builddir = $filtered_argv[2];
     221  my $indexdir      = $filtered_argv[3];
    235222
    236223  # We only need the Solr handle opened if we are indexing the
    237224  # documents, not if we are just storing the text
    238225  if ($mode eq "index") {
    239     open_java_solr($collect, $ds_idx, $full_builddir, $indexdir, $removeold);
     226    open_java_solr($core, $full_builddir, $indexdir);
    240227  }
    241228
  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r24497 r24501  
    112112    else
    113113    {
    114     my $collection = $self->{'collection'};
     114    my $site        = $self->{'site'};
     115    my $collect     = $self->{'collection'};   
     116    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
     117    my $core        = $core_prefix; # unused in this call to solr_passes
    115118
    116119        print STDERR "Executable:    $solr_passes_exe\n";
    117120        print STDERR "Sections:      $solr_passes_sections\n";
    118121        print STDERR "Build Dir:     $build_dir\n";
    119         print STDERR "Cmd:           $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra\n";
    120     if (!open($handle, "| $solr_passes_exe $collection text dummy \"$build_dir\" \"dummy\"   $osextra"))
     122        print STDERR "Cmd:           $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra\n";
     123    if (!open($handle, "| $solr_passes_exe $core text \"$build_dir\" \"dummy\"   $osextra"))
    121124    {
    122125        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
     
    243246       
    244247    my $solr_home = $ENV{'GEXT_SOLR'};
    245     my $in_dirname = &util::filename_cat($solr_home,"etc","conf");
     248##    my $in_dirname = &util::filename_cat($solr_home,"etc","conf");
     249    my $in_dirname = &util::filename_cat($solr_home,"conf");
    246250    my $schema_in_filename = &util::filename_cat($in_dirname,"schema.xml.in");
    247251
     
    281285    # it up, and only returns when the server is "reading and listening"
    282286 
    283     my $solr_server = new solrserver();
     287    my $solr_server = new solrserver($self->{'build_dir'});
    284288    $solr_server->start();
    285289    $self->{'solr_server'} = $solr_server;
     
    364368    # => at most two cores <colname>-Doc and <colname>-Sec
    365369
    366     my $collection = $self->{'collection'};
     370    my $site        = $self->{'site'};
     371    my $collect     = $self->{'collection'};
     372    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
    367373
    368374    # my $idx = $self->{'index_mapping'}->{$index};
    369375    my $idx = "idx";
    370376
    371     my $site = $self->{'site'};
     377    my $build_dir = $self->{'build_dir'};
    372378
    373379    foreach my $level (keys %{$self->{'levels'}}) {
    374380   
    375381    my ($pindex) = $level =~ /^(.)/;
    376    
    377     my $core = $collection."-".$pindex.$idx;
    378 
    379     # prefix site if exists (e.g. Greenstone 3)
    380     $core = "$site-$core" if defined $site;
     382
     383    my $index_dir = $pindex.$idx;
     384    my $core = "$core_prefix-$index_dir";
     385
     386    my $force_removeold = ($self->{'incremental'}) ? 0 : 1;
     387    if ($force_removeold) {
     388        print $outhandle "\n-removeold set (new index will be created)\n";
     389
     390        my $full_index_dir = &util::filename_cat($build_dir,$index_dir);
     391        &util::rm_r($full_index_dir);
     392        &util::mk_dir($full_index_dir);
     393    }
    381394
    382395    # if collect==core already in solr.xml (check with STATUS)
     
    421434    my $solr_passes_sections = $llevel;
    422435
    423     my $opt_create_index = ($self->{'incremental'}) ? "" : "-removeold";
    424 
    425436    my $osextra = "";
    426437    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
     
    478489    $handle = *STDOUT;
    479490    } else {
    480     my $collection = $self->{'collection'};
    481     my $ds_idx = $self->{'index_mapping'}->{$index};
    482 
    483     print STDERR "Cmd: $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra\n";
    484     if (!open($handle, "| $solr_passes_exe $opt_create_index $collection index $ds_idx \"$build_dir\" \"$indexdir\"   $osextra")) {
     491    my $site        = $self->{'site'};
     492    my $collect     = $self->{'collection'};
     493    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
     494    my $ds_idx      = $self->{'index_mapping'}->{$index};
     495    my $core        = "$core_prefix-$ds_idx";
     496
     497    print STDERR "Cmd: $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra\n";
     498    if (!open($handle, "| $solr_passes_exe $core index \"$build_dir\" \"$indexdir\"   $osextra")) {
    485499        print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
    486500        die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n";
  • gs3-extensions/solr/trunk/src/perllib/solrserver.pm

    r24486 r24501  
    3434sub new {
    3535    my $class = shift(@_);
     36    my ($build_dir) = @_;
    3637
    3738    my $self = { 'jetty_stop_key' => "greenstone-solr" };
     39
     40    $self->{'build_dir'} = $build_dir;
    3841
    3942    my $search_path = &solrutil::get_search_path();
     
    7477    my $in_preamble = ($output_format eq "xml") ? 1 : 0;
    7578   
     79##    print STDERR "**** wgetcmd = \n $cmd\n";
     80
    7681    if (open(WIN,"$cmd |")) {
    7782
     
    213218{
    214219    my $self = shift @_;
    215     my ($core) = @_;
    216 
    217     my ($ds_idx) = ($core =~ m/^.*-(.*)$/);
     220    my ($core,$removeold) = @_;
     221
     222    my ($ds_idx) = ($core =~ m/^.*-(.*?)$/);
    218223
    219224    my $cgi_get_args = "action=CREATE&name=$core";
     
    243248    chdir($solr_home);
    244249   
    245     my $solr_etc = &util::filename_cat($solr_home,"etc");
     250##    my $solr_etc = &util::filename_cat($solr_home,"etc");
    246251
    247252    my $server_props = "-DSTOP.PORT=$jetty_stop_port";
    248253    $server_props .= " -DSTOP.KEY=".$self->{'jetty_stop_key'};
    249     $server_props .= " -Dsolr.solr.home=$solr_etc";
     254    $server_props .= " -Dsolr.solr.home=$solr_home";
    250255
    251256    my $full_server_jar = $self->{'full_server_jar'};
     
    253258    my $server_java_cmd = "java $server_props -jar \"$full_server_jar\"";
    254259
    255 ##    print STDERR "**** server cmd start = $server_java_cmd\n";
    256260
    257261    my $server_status = "unknown";
     
    261265    }
    262266    elsif (open(STARTIN,"$server_java_cmd 2>&1 |")) {
     267
     268##  print STDERR "**** startup up server with cmd start =\n $server_java_cmd\n";
    263269
    264270    my $line;
  • gs3-extensions/solr/trunk/src/perllib/solrutil.pm

    r24483 r24501  
    6363sub open_post_pipe
    6464{
    65     my ($collect,$ds_idx) = @_;
     65    my ($core) = @_;
    6666
    6767    my $search_path = get_search_path();
     
    7575   
    7676    # Now run solr-post command
    77     my $core = $collect."-".$ds_idx;
    7877    my $post_props = "-Durl=http://localhost:$jetty_port/solr/$core/update";
    7978    $post_props .= " -Ddata=stdin";
     
    8281    my $post_java_cmd = "java $post_props -jar \"$full_post_jar\"";
    8382   
    84 ###  print STDERR "**** post cmd = $post_java_cmd\n";
     83##  print STDERR "**** post cmd = $post_java_cmd\n";
    8584   
    8685    open (PIPEOUT, "| $post_java_cmd")
Note: See TracChangeset for help on using the changeset viewer.