Changeset 33725

Timestamp:

2019-11-28T14:10:27+13:00 (4 years ago)

Author:

kjdon

Message:

tidied this up a bit with regards to solr core activation and deactivation. when unloading cores, should always look for didx and sidx - if the collection was built previously with both doc and sec indexes, then the user changes to just one of these, we need to delete teh old one. Also, if the collection is not solr, but there is an etc/conf folder, then it was solr last build. Delete any solr cores there are, and then delete the conf folder.

File:

• main/trunk/greenstone2/bin/script/activate.pl (modified) (9 diffs)

main/trunk/greenstone2/bin/script/activate.pl

@@ -189 +189 @@
     $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir);
     $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir);
-
+    my $solr_conf_dir = &FileUtils::filenameConcatenate($collection_dir, "etc", "conf");
+    
     my $gsserver = new servercontrol($qualified_collection, $site, $default_verbosity, $build_dir, $index_dir, $collect_dir, $library_url, $library_name);
 
@@ -256 +257 @@
     $gsserver->do_deactivate() unless $skipactivation;    
 
-    # 2b. If we're working with a solr collection, then start up the solrserver now.
+    # solr core reloading - previously for incremental building, the solr core is unloaded and created
+    # - why is this? Does this actually need to happen? The following assumes that we *do* want to
+    # unload and create the core even if incremental building
+    
+    # solr deactivation. Are we a solr collection? unload any cores that are
+    # present for this collection
+    # If we are not a solr collection, is there etc/solr/conf?
+    # if so, we were a solr collection previously, need to unload cores.
+    # in both cases, we attempt to unload didx and sidx cores, as well
+    # as both building cores
+    
+    my $collection_was_previously_solr = 0;
+    if ($buildtype ne "solr" && FileUtils::directoryExists($solr_conf_dir)) {
+    $collection_was_previously_solr = 1;
+    }
+    
+    # 2b. If we're working with a solr collection, then start up the solrserver now and unload 
+    # any cores that may exist for this collection (building and index cores)
     my $solr_server;
     my @corenames = ();
-    if($buildtype eq "solr") { # start up the jetty server  
+    my @all_cores = ();
+    my $core_basename = "";
+
+    # for unloading, lets try unloading both didx and sidx as these may have
+    # both been there previously, and if the user removed one in the config file
+    # we won't know that.
+    if($buildtype eq "solr" || $collection_was_previously_solr) { 
+
     my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl
     unshift (@INC, "$solr_ext/perllib");
@@ -267 +292 @@
     # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead
     my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection);
-
-    # See solrbuilder.pm to get the indexing levels (document, section) from the collectcfg file
-    # Used to generate core names from them and remove cores by name
-    foreach my $level ( @{$collectcfg->{'levels'}} ){
-        my ($pindex) = $level =~ /^(.)/;
-        my $indexname = $pindex."idx";
-        push(@corenames, "$site-$colname-$indexname"); #"$site-$colname-didx", "$site-$colname-sidx"
-        }
-    
-    # If the Solr/Jetty server is not already running, the following starts
+    $core_basename = "$site-$colname";
+    # all_cores contains both sidx and didx
+    @all_cores = ("$core_basename-didx", "$core_basename-sidx");
+    
+    # If the Solr server is not already running, the following starts
     # it up, and only returns when the server is "ready and listening"  
     $solr_server = new solrserver($build_dir);
     $solr_server->start();
-    
-    # We'll be moving building to index. For a solr collection, there's further 
-    # special processing to make a corresponding change to the solr.xml
-    # by removing the temporary building cores and (re)creating the index cores
-    }
-
-
+
+    # unload all the possible cores
+    foreach my $corename (@all_cores) {
+        $solr_server->admin_unload_core("$corename") if ($solr_server->admin_ping_core("$corename"));
+        $solr_server->admin_unload_core("building-$corename") if ($solr_server->admin_ping_core("building-$corename"));
+    }
+
+    # tidy up other files if we were solr but are not anymore
+    if ($collection_was_previously_solr) {
+        # regenerate the solr.xml.in file to reflect the changes to cores
+        $solr_server->solr_xml_to_solr_xml_in();
+        # we are finished with solrserver now, stop it if necessary
+        if ($solr_server->explicitly_started()) {
+        $solr_server->stop();
+        }
+        # lets delete the conf folder as we are no longer solr
+        &FileUtils::removeFilesRecursive($solr_conf_dir);
+    }
+
+    }
+    
     # 3. Do all the moving building to index stuff now  
     
@@ -292 +326 @@
     # If keepold: move building's contents into index, where only duplicates will get deleted.
     # removeold and keepold can't both be on at the same time
-    # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does (moving building to index)
+    # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does 
+    # (moving building to index)
     my $incremental_mode;
-    ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, 
-                           $incremental,
-                           $index_dir, # checkdir. Usually archives or export to be deleted. activate.pl deletes index
-                           $collectcfg);
+    ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, $incremental, $index_dir, $collectcfg);
     
     if($removeold) {
@@ -303 +335 @@
     if(&FileUtils::directoryExists($index_dir)) {
         $gsserver->print_task_msg("Removing \"index\"");
-        
-        if ($buildtype eq "solr") {
-        # if solr, remove any cores that are using the index_dir before deleting this dir
-        foreach my $corename (@corenames) {
-            $solr_server->admin_unload_core($corename) if ($solr_server->admin_ping_core($corename));
-        }
-        }   
         
         &FileUtils::removeFilesRecursive($index_dir);
@@ -320 +345 @@
         }
     }
-    
-    # if remote GS server: gliserver.pl would call activate.pl to activate 
-    # the collection at this point since activate.pl lives on the server side
-    
-    if ($buildtype eq "solr") {
-        # if solr, remove any cores that are using the building_dir before moving this dir onto index
-        foreach my $corename (@corenames) {
-        $solr_server->admin_unload_core("building-$corename") if ($solr_server->admin_ping_core("building-$corename"));
-        }
-    }
-    
+        
     # Move the building directory to become the new index directory
     $gsserver->print_task_msg("Moving \"building\" -> \"index\"");
@@ -338 +353 @@
     }
     }
+    
     elsif ($keepold || $incremental) {
-    if ($buildtype eq "solr" && $build_dir ne $index_dir) {
-        # if solr, remove any cores that may be using the building_dir before moving this dir onto index
-        foreach my $corename (@corenames) {         
-        $solr_server->admin_unload_core("building-$corename") if $solr_server->admin_ping_core("building-$corename");
-        }
-    }
     
     if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl
@@ -364 +374 @@
     if ($buildtype eq "solr") {
     # Call CREATE action to get the old cores pointing to the index folder
+    #  -- any building or index cores have been unloaded already
+    #  -- load up the new one
+    
+    # generate actual core names
+    foreach my $level ( @{$collectcfg->{'levels'}} ){
+        my ($pindex) = $level =~ /^(.)/;
+        my $indexname = $pindex."idx";
+        push(@corenames, "$core_basename-$indexname"); 
+    }
+    
     foreach my $corename (@corenames) {
-        if($removeold) {
-        # Call CREATE action to get all cores pointing to the index folder, since building is now index
-        $solr_server->admin_create_core($corename, $index_dir);
-        
-        } elsif ($keepold || $incremental) { 
-        # Call RELOAD core. Should already be using the index_dir directory for $keepold and $incremental case
-        
-        # Ping to see if corename exists, if it does, unload and create (avoid reload due to Solr memory leak on reload)
-        if ($solr_server->admin_ping_core($corename)) {
-            $solr_server->admin_unload_core($corename); 
-        } 
-        $solr_server->admin_create_core($corename, $index_dir);
-        
-        }
+        $solr_server->admin_create_core($corename, $index_dir);
     }
     
     # regenerate the solr.xml.in from solr.xml in case we are working off a dvd. 
     $solr_server->solr_xml_to_solr_xml_in();
+    # stop the server if necessary
+    if ($solr_server->explicitly_started()) {
+        $solr_server->stop();
+    }
+
     }
 
@@ -400 +412 @@
     $gsserver->print_msg("\n");
     
-    if($buildtype eq "solr") {
-    if ($solr_server->explicitly_started()) {
-        $solr_server->stop();
-    }
-    }
 }