Ignore:
Timestamp:
2023-06-21T10:05:43+12:00 (6 months ago)
Author:
kjdon
Message:

code updated to support subcollections and language partitions. we don't assume just idx anymore, we use the index defs which includes subcoll info. store the corenmames in buildcfg, for use later.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • gs3-extensions/solr/trunk/src/perllib/solrbuilder.pm

    r35685 r37787  
    219219  my @indexfields = ();
    220220
     221  # so we don't add duplicates - now that we have subcollections working, there will be multiple index defs with the same fields (and different subcolls)
     222  my $done_fields = {};
    221223  # @todo support: $self->{'buildproc'}->{'extraindexfields'}
    222   foreach my $fields (@{$self->{'collect_cfg'}->{'indexes'}})
     224  foreach my $origfields (@{$self->{'collect_cfg'}->{'indexes'}})
    223225  {
    224     # remove subcoll stuff
     226      # remove subcoll stuff for finding fields, but we need to leave it in the oroginal index definition for later, so make a copy
     227      my $fields = $origfields;
    225228    $fields =~ s/:.*$//;
    226229    foreach my $field (split(';', $fields))
    227230    {
     231    next if (defined $done_fields->{$field});
    228232      my $shortname = 'ERROR';
    229233      if ($field eq 'allfields')
    … …  
    241245      push (@indexfieldmap, $field . '->' . $shortname);
    242246      push (@indexfields, $field);
     247    $done_fields->{$field} = 1;
    243248    }
    244249  }
    … …  
    254259  }
    255260}
     261
    256262
    257263# Generate solr schema.xml file based on indexmapfield and other associated
    … …  
    280286
    281287    foreach my $ifm (@{$self->{'build_cfg'}->{'indexfieldmap'}}) {
    282 
    283288        my ($fullfieldname, $field) = ($ifm =~ m/^(.*)->(.*)$/);
    284289
    … …  
    415420    }
    416421
     422    # create the mapping between the index descriptions
     423    # and their directory names (includes subcolls and langs)
     424    $self->{'index_mapping'} = $self->create_index_mapping ($indexes);
     425
     426
    417427    # skip para-level check, as this is done in the main 'build_indexes'
    418428    # routine
    … …  
    484494    #
    485495    $self->make_final_field_list();
     496
    486497    $self->premake_solr_auxiliary_files();
    487498
    … …  
    492503    my $collect     = $self->{'collection'};
    493504    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;
    494 
    495     # my $idx = $self->{'index_mapping'}->{$index};
    496     my $idx = "idx";
    497 
    498505    my $build_dir = $self->{'build_dir'};
    499506
    500     foreach my $level (keys %{$self->{'levels'}}) {
     507    $self->{'solrcores'} = [];
     508    foreach my $index (@$indexes) {
     509    if ($self->want_built($index)) {
     510
     511        my $idx = $self->{'index_mapping'}->{$index};
     512       
     513        foreach my $level (keys %{$self->{'levels'}}) {
    501514   
    502     my ($pindex) = $level =~ /^(.)/;
    503 
    504     my $index_dir = $pindex.$idx;
    505     my $core = "$core_prefix-$index_dir";
    506 
    507     # force_removeold == opposite of being run in 'incremental' mode
    508     my $force_removeold = ($self->{'incremental'}) ? 0 : 1;
    509 
    510     if ($force_removeold) {
    511         print $outhandle "\n-removeold set (new index will be created)\n";
    512 
    513         # create cores under temporary core names, corresponding to building directory
    514         $core = "building-".$core;
    515 
    516         my $full_index_dir = &FileUtils::filenameConcatenate($build_dir,$index_dir);
    517         &FileUtils::removeFilesRecursive($full_index_dir);
    518         &FileUtils::makeDirectory($full_index_dir);
    519 
    520         my $full_tlog_dir = &FileUtils::filenameConcatenate($full_index_dir, "tlog");
    521         &FileUtils::makeDirectory($full_tlog_dir);
    522 
    523         # Solr then wants an "index" folder within this general index area!
     515        my ($pindex) = $level =~ /^(.)/;
     516       
     517        my $index_dir = $pindex.$idx;
     518        my $core = "$core_prefix-$index_dir";
     519        push (@{$self->{'solrcores'}}, $index_dir);
     520        # force_removeold == opposite of being run in 'incremental' mode
     521        my $force_removeold = ($self->{'incremental'}) ? 0 : 1;
     522       
     523        if ($force_removeold) {
     524            print $outhandle "\n-removeold set (new index will be created)\n";
     525           
     526            # create cores under temporary core names, corresponding to building directory
     527            $core = "building-".$core;
     528           
     529            my $full_index_dir = &FileUtils::filenameConcatenate($build_dir,$index_dir);
     530            &FileUtils::removeFilesRecursive($full_index_dir);
     531            &FileUtils::makeDirectory($full_index_dir);
     532           
     533            my $full_tlog_dir = &FileUtils::filenameConcatenate($full_index_dir, "tlog");
     534            &FileUtils::makeDirectory($full_tlog_dir);
     535           
     536            # Solr then wants an "index" folder within this general index area!
    524537#       my $full_index_index_dir = &FileUtils::filenameConcatenate($full_index_dir,"index");
    525538#       &FileUtils::makeDirectory($full_index_index_dir);
    526 
    527 
    528         # now go on and create new index
    529         print $outhandle "Creating Solr core: $core\n";
    530         $solr_server->admin_create_core($core);
    531 
    532     }
    533     else {
    534         # if collect==core is already in solr.xml (check with STATUS)
    535         # => use RELOAD* call to refresh fields now expressed in schema.xml
    536         #
    537         # else
    538         # => use CREATE API to add to solr.xml
    539         #
    540         # No longer calling RELOAD, because Georgy documented a memory leak with it (svn r32178)
    541         # Using unload + create to get the same effect as RELOAD without its side-effects.
    542         #
    543        
    544         my $check_core_exists = $solr_server->admin_ping_core($core);
    545        
    546         if ($check_core_exists) {       
    547         print $outhandle "Unloading Solr core: $core\n";
    548         $solr_server->admin_unload_core($core);
    549         }
    550        
    551         print $outhandle "Creating Solr core: $core\n";
    552         $solr_server->admin_create_core($core);
    553        
    554     }
    555     }
     539           
     540           
     541            # now go on and create new index
     542            print $outhandle "Creating Solr core: $core\n";
     543            $solr_server->admin_create_core($core);
     544           
     545        }
     546        else {
     547            # if collect==core is already in solr.xml (check with STATUS)
     548            # => use RELOAD* call to refresh fields now expressed in schema.xml
     549            #
     550            # else
     551            # => use CREATE API to add to solr.xml
     552            #
     553            # No longer calling RELOAD, because Georgy documented a memory leak with it (svn r32178)
     554            # Using unload + create to get the same effect as RELOAD without its side-effects.
     555            #
     556           
     557            my $check_core_exists = $solr_server->admin_ping_core($core);
     558           
     559            if ($check_core_exists) {       
     560            print $outhandle "Unloading Solr core: $core\n";
     561            $solr_server->admin_unload_core($core);
     562            }
     563           
     564            print $outhandle "Creating Solr core: $core\n";
     565            $solr_server->admin_create_core($core);
     566        }
     567        } # foreach level
     568    } #if (want build index)
     569    } #foreach index    `
    556570
    557571}
    … …  
    737751    # worry about the needed symmetry at start and end of buildcol to create building- cores
    738752    # in symmetry with unloading them here.
    739    
     753
     754    # update: now that we can do subcollections with solr, we don't know what cores may have been there
     755    # lets just removes all cores for the collection.
    740756    my $site        = $self->{'site'};
    741757    my $collect     = $self->{'collection'};
    742758    my $core_prefix = (defined $site) ? "$site-$collect" : $collect;   
    743759    my $build_dir = $self->{'build_dir'};
    744     my $idx = "idx";
    745760   
    746     foreach my $level (keys %{$self->{'levels'}}) {
    747    
    748     my ($pindex) = $level =~ /^(.)/;
    749 
    750     my $index_dir = $pindex.$idx;
    751     my $corename = "building-$core_prefix-$index_dir";
    752 
    753     # unload the core if it exists. Since I can't think of exactly in which cases
    754     # the building-cores exist and won't exist (e.g. removeold), I'll just always
    755     # first check if the building-core exists and then unload it.
    756     if ($solr_server->admin_ping_core($corename)) {
    757         ##print STDERR "@@@@ solrbuilder::post_build_indexes(): Now unloading this index's building core: $corename\n\n";
    758         $solr_server->admin_unload_core_explicitly_retaining_index($corename)
    759     }
    760 
    761     }
    762    
     761    $solr_server->admin_unload_all_cores_for_prefix("building-$core_prefix");
     762
    763763    # Also need to stop the Solr server (be it tomcat or jetty) if it was explicitly started
    764764    # in pre_build_indexes()
    … …  
    792792    $build_cfg->{'indexfacetfields'} = \@facetfields;
    793793    $build_cfg->{'indexfacetfieldmap'} = \@facetfieldmap;
     794    # store the core names in buildConfig, so that activate.pl can use them,
     795    $build_cfg->{'solrcores'} = $self->{'solrcores'};
     796
    794797} 
    7957981;
Note: See TracChangeset for help on using the changeset viewer.