Changeset 33725

Show
Ignore:
Timestamp:
28.11.2019 14:10:27 (9 days ago)
Author:
kjdon
Message:

tidied this up a bit with regards to solr core activation and deactivation. when unloading cores, should always look for didx and sidx - if the collection was built previously with both doc and sec indexes, then the user changes to just one of these, we need to delete teh old one. Also, if the collection is not solr, but there is an etc/conf folder, then it was solr last build. Delete any solr cores there are, and then delete the conf folder.

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/bin/script/activate.pl

    r33392 r33725  
    189189    $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir); 
    190190    $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir); 
    191  
     191    my $solr_conf_dir = &FileUtils::filenameConcatenate($collection_dir, "etc", "conf"); 
     192     
    192193    my $gsserver = new servercontrol($qualified_collection, $site, $default_verbosity, $build_dir, $index_dir, $collect_dir, $library_url, $library_name); 
    193194 
     
    256257    $gsserver->do_deactivate() unless $skipactivation;     
    257258 
    258     # 2b. If we're working with a solr collection, then start up the solrserver now. 
     259    # solr core reloading - previously for incremental building, the solr core is unloaded and created 
     260    # - why is this? Does this actually need to happen? The following assumes that we *do* want to 
     261    # unload and create the core even if incremental building 
     262     
     263    # solr deactivation. Are we a solr collection? unload any cores that are 
     264    # present for this collection 
     265    # If we are not a solr collection, is there etc/solr/conf? 
     266    # if so, we were a solr collection previously, need to unload cores. 
     267    # in both cases, we attempt to unload didx and sidx cores, as well 
     268    # as both building cores 
     269     
     270    my $collection_was_previously_solr = 0; 
     271    if ($buildtype ne "solr" && FileUtils::directoryExists($solr_conf_dir)) { 
     272    $collection_was_previously_solr = 1; 
     273    } 
     274     
     275    # 2b. If we're working with a solr collection, then start up the solrserver now and unload  
     276    # any cores that may exist for this collection (building and index cores) 
    259277    my $solr_server; 
    260278    my @corenames = (); 
    261     if($buildtype eq "solr") { # start up the jetty server   
     279    my @all_cores = (); 
     280    my $core_basename = ""; 
     281 
     282    # for unloading, lets try unloading both didx and sidx as these may have 
     283    # both been there previously, and if the user removed one in the config file 
     284    # we won't know that. 
     285    if($buildtype eq "solr" || $collection_was_previously_solr) {  
     286 
    262287    my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl 
    263288    unshift (@INC, "$solr_ext/perllib"); 
     
    267292    # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead 
    268293    my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection); 
    269  
    270     # See solrbuilder.pm to get the indexing levels (document, section) from the collectcfg file 
    271     # Used to generate core names from them and remove cores by name 
    272     foreach my $level ( @{$collectcfg->{'levels'}} ){ 
    273         my ($pindex) = $level =~ /^(.)/; 
    274         my $indexname = $pindex."idx"; 
    275         push(@corenames, "$site-$colname-$indexname"); #"$site-$colname-didx", "$site-$colname-sidx" 
    276         } 
    277      
    278     # If the Solr/Jetty server is not already running, the following starts 
     294    $core_basename = "$site-$colname"; 
     295    # all_cores contains both sidx and didx 
     296    @all_cores = ("$core_basename-didx", "$core_basename-sidx"); 
     297     
     298    # If the Solr server is not already running, the following starts 
    279299    # it up, and only returns when the server is "ready and listening"   
    280300    $solr_server = new solrserver($build_dir); 
    281301    $solr_server->start(); 
    282      
    283     # We'll be moving building to index. For a solr collection, there's further  
    284     # special processing to make a corresponding change to the solr.xml 
    285     # by removing the temporary building cores and (re)creating the index cores 
    286     } 
    287  
    288  
     302 
     303    # unload all the possible cores 
     304    foreach my $corename (@all_cores) { 
     305        $solr_server->admin_unload_core("$corename") if ($solr_server->admin_ping_core("$corename")); 
     306        $solr_server->admin_unload_core("building-$corename") if ($solr_server->admin_ping_core("building-$corename")); 
     307    } 
     308 
     309    # tidy up other files if we were solr but are not anymore 
     310    if ($collection_was_previously_solr) { 
     311        # regenerate the solr.xml.in file to reflect the changes to cores 
     312        $solr_server->solr_xml_to_solr_xml_in(); 
     313        # we are finished with solrserver now, stop it if necessary 
     314        if ($solr_server->explicitly_started()) { 
     315        $solr_server->stop(); 
     316        } 
     317        # lets delete the conf folder as we are no longer solr 
     318        &FileUtils::removeFilesRecursive($solr_conf_dir); 
     319    } 
     320 
     321    } 
     322     
    289323    # 3. Do all the moving building to index stuff now   
    290324     
     
    292326    # If keepold: move building's contents into index, where only duplicates will get deleted. 
    293327    # removeold and keepold can't both be on at the same time 
    294     # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does (moving building to index) 
     328    # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does  
     329    # (moving building to index) 
    295330    my $incremental_mode; 
    296     ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold,  
    297                            $incremental, 
    298                            $index_dir, # checkdir. Usually archives or export to be deleted. activate.pl deletes index 
    299                            $collectcfg); 
     331    ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, $incremental, $index_dir, $collectcfg); 
    300332     
    301333    if($removeold) { 
     
    303335    if(&FileUtils::directoryExists($index_dir)) { 
    304336        $gsserver->print_task_msg("Removing \"index\""); 
    305          
    306         if ($buildtype eq "solr") { 
    307         # if solr, remove any cores that are using the index_dir before deleting this dir 
    308         foreach my $corename (@corenames) { 
    309             $solr_server->admin_unload_core($corename) if ($solr_server->admin_ping_core($corename)); 
    310         } 
    311         }    
    312337         
    313338        &FileUtils::removeFilesRecursive($index_dir); 
     
    320345        } 
    321346    } 
    322      
    323     # if remote GS server: gliserver.pl would call activate.pl to activate  
    324     # the collection at this point since activate.pl lives on the server side 
    325      
    326     if ($buildtype eq "solr") { 
    327         # if solr, remove any cores that are using the building_dir before moving this dir onto index 
    328         foreach my $corename (@corenames) { 
    329         $solr_server->admin_unload_core("building-$corename") if ($solr_server->admin_ping_core("building-$corename")); 
    330         } 
    331     } 
    332      
     347         
    333348    # Move the building directory to become the new index directory 
    334349    $gsserver->print_task_msg("Moving \"building\" -> \"index\""); 
     
    338353    } 
    339354    } 
     355     
    340356    elsif ($keepold || $incremental) { 
    341     if ($buildtype eq "solr" && $build_dir ne $index_dir) { 
    342         # if solr, remove any cores that may be using the building_dir before moving this dir onto index 
    343         foreach my $corename (@corenames) {          
    344         $solr_server->admin_unload_core("building-$corename") if $solr_server->admin_ping_core("building-$corename"); 
    345         } 
    346     } 
    347357     
    348358    if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl 
     
    364374    if ($buildtype eq "solr") { 
    365375    # Call CREATE action to get the old cores pointing to the index folder 
     376    #  -- any building or index cores have been unloaded already 
     377    #  -- load up the new one 
     378     
     379    # generate actual core names 
     380    foreach my $level ( @{$collectcfg->{'levels'}} ){ 
     381        my ($pindex) = $level =~ /^(.)/; 
     382        my $indexname = $pindex."idx"; 
     383        push(@corenames, "$core_basename-$indexname");  
     384    } 
     385     
    366386    foreach my $corename (@corenames) { 
    367         if($removeold) { 
    368         # Call CREATE action to get all cores pointing to the index folder, since building is now index 
    369         $solr_server->admin_create_core($corename, $index_dir); 
    370          
    371         } elsif ($keepold || $incremental) {  
    372         # Call RELOAD core. Should already be using the index_dir directory for $keepold and $incremental case 
    373          
    374         # Ping to see if corename exists, if it does, unload and create (avoid reload due to Solr memory leak on reload) 
    375         if ($solr_server->admin_ping_core($corename)) { 
    376             $solr_server->admin_unload_core($corename);  
    377         }  
    378         $solr_server->admin_create_core($corename, $index_dir); 
    379          
    380         } 
     387        $solr_server->admin_create_core($corename, $index_dir); 
    381388    } 
    382389     
    383390    # regenerate the solr.xml.in from solr.xml in case we are working off a dvd.  
    384391    $solr_server->solr_xml_to_solr_xml_in(); 
     392    # stop the server if necessary 
     393    if ($solr_server->explicitly_started()) { 
     394        $solr_server->stop(); 
     395    } 
     396 
    385397    } 
    386398 
     
    400412    $gsserver->print_msg("\n"); 
    401413     
    402     if($buildtype eq "solr") { 
    403     if ($solr_server->explicitly_started()) { 
    404         $solr_server->stop(); 
    405     } 
    406     } 
    407414} 
    408415