Changeset 30517


Ignore:
Timestamp:
2016-05-09T16:53:48+12:00 (5 years ago)
Author:
ak19
Message:

Fixing incremental-rebuild when the database is gdbm. At this point (see buildcolutils.pm), the code needs to deactivate the collection before calling make_infodatabase(), since otherwise there's a lock on the gdbm database which prevents successful incremental-rebuild and activation.

Location:
main/trunk/greenstone2
Files:
8 edited

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/bin/script/activate.pl

    r30513 r30517  
    465465}
    466466
    467 
    468 # Most of the arguments are familiar from the building scripts like buildcol.pl
    469 # The special optional argument -library_url is for when we're dealing with a web
    470 # library server such as an apache that's separate from any included with GS2.
    471 # In such a case, this script's caller should pass in -library_url <URL>.
    472 #
    473 # $site argument must be specified in the cmdline for collectionConfig.xml to get
    474 # generated which makes $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
    475 sub main
    476 {
    477     my ($argc,@argv) = @_;
    478 
    479     if (($argc==0)  || (($argc==1) && ($argv[0] =~ m/^--?h(elp)?$/))) {
    480     my ($progname) = ($0 =~ m/^.*[\/|\\](.*?)$/);
    481    
    482    
    483     print STDERR "\n";
    484     print STDERR "Usage: $progname [-collectdir c -builddir b -indexdir i -site s -skipactivation -removeold -keepold -verbosity v\n";
    485     print STDERR "\t-library_url URL -library_name n] <[colgroup/]collection>\n";
    486     print STDERR "\n";
    487    
    488     exit(-1);
    489     }
    490    
    491     # get the collection details
    492     my $qualified_collection = pop @argv; # qualified collection
    493    
    494     my $collect_dir = undef; #"collect"; # can't be "collect" when only -site is provided for GS3
    495     my $build_dir = undef;
    496     my $index_dir = undef;
    497     my $site = undef;
    498    
    499     # if run from server (java code), it will handle deactivation and activation to prevent open file handles when java launches this script and exits:
    500     my $skipactivation = 0;
    501     my $removeold = 0;
    502     my $keepold = 0;
    503     my $incremental = 0; # used by solr
    504    
    505     my $library_url = $ENV{'GSDL_LIBRARY_URL'} || undef; # to be specified on the cmdline if not using a GS-included web server
    506     # the GSDL_LIBRARY_URL env var is useful when running cmdline buildcol.pl in the linux package manager versions of GS3
    507    
    508     my $library_name = undef;
    509    
    510     while (my $arg = shift @argv) {
    511     if ($arg eq "-collectdir") {
    512         $collect_dir = shift @argv;
    513     }
    514     elsif ($arg eq "-builddir") {
    515         $build_dir = shift @argv;
    516     }
    517     elsif ($arg eq "-indexdir") {
    518         $index_dir = shift @argv;
    519     }
    520     elsif ($arg eq "-site") {
    521         $site = shift @argv;
    522     }
    523     elsif ($arg eq "-skipactivation") {
    524         $skipactivation = 1;
    525     }
    526     elsif ($arg eq "-removeold") {
    527         $removeold = 1;
    528     }
    529     elsif ($arg eq "-keepold") {
    530         $keepold = 1;
    531     }
    532     elsif ($arg eq "-incremental") {
    533         $incremental = 1;
    534     }
    535     elsif ($arg eq "-library_url") {
    536         $library_url = shift @argv;
    537     }
    538     elsif ($arg eq "-library_name") {
    539         $library_name = shift @argv;
    540     }
    541     elsif ($arg eq "-verbosity") {
    542         $default_verbosity = shift @argv; # global variable
    543        
    544         # ensure we're working with ints not strings (int context not str context), in case verbosity=0
    545         # http://stackoverflow.com/questions/288900/how-can-i-convert-a-string-to-a-number-in-perl
    546         $default_verbosity = int($default_verbosity || 0); ### is this the best way?
    547     }
    548     }
    549    
    550     # work out the building and index dirs
    551     my $collection_dir = &util::resolve_collection_dir($collect_dir, $qualified_collection, $site);
    552     $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir);
    553     $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir);
    554    
    555     &print_task_msg("Running  Collection  Activation  Stage");
    556    
    557     # get and check the collection name
    558     if ((&colcfg::use_collection($site, $qualified_collection, $collect_dir)) eq "") {
    559     &print_msg("Unable to use collection \"$qualified_collection\" within \"$collect_dir\"\n");
    560     exit -1;
    561     }
    562    
    563     # Read in the collection configuration file.
    564     # Beware: Only if $site is specified in the cmdline does collectionConfig.xml get
    565     # generated and does $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
    566     my $gs_mode = "gs2";
    567     if ((defined $site) && ($site ne "")) { # GS3
    568     $gs_mode = "gs3";
    569     }
    570     my $collect_cfg_filename = &colcfg::get_collect_cfg_name(STDERR, $gs_mode);
    571     my $collectcfg = &colcfg::read_collection_cfg ($collect_cfg_filename,$gs_mode);
    572    
    573     # look for build.cfg/buildConfig.xml
    574     my $build_cfg_filename ="";
    575    
    576     if ($gs_mode eq "gs2") {
    577     $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir,"build.cfg");
    578     } else {
    579     $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir, "buildConfig.xml");
    580     # gs_mode is GS3. Set the site now if this was not specified as cmdline argument
    581     #$site = "localsite" unless defined $site;
    582     }
    583    
    584     # We need to know the buildtype for Solr.
    585     # Any change of indexers is already detected and handled by the calling code (buildcol or
    586     # full-rebuild), so that at this stage the config file's buildtype reflects the actual buildtype.
    587    
    588     # From buildcol.pl we use searchtype for determining buildtype, but for old versions, use buildtype
    589     my $buildtype;
    590     if (defined $collectcfg->{'buildtype'}) {
    591     $buildtype = $collectcfg->{'buildtype'};
    592     } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
    593     $buildtype = "mgpp";
    594     } else {
    595     $buildtype = "mg"; #mg is the default
    596     }
    597    
    598     # can't do anything without a build directory with something in it to move into index
    599     # Except if we're (doing incremental) building for solr, where we want to still
    600     # activate and deactivate collections including for the incremental case
    601     if(!&FileUtils::directoryExists($build_dir)) {
    602     &print_msg("No building folder at $build_dir to move to index.\n");
    603     exit -1 unless ($buildtype eq "solr"); #&& $incremental);
    604     } elsif (&FileUtils::isDirectoryEmpty($build_dir)) {
    605     &print_msg("Nothing in building folder $build_dir to move into index folder.\n");
    606     exit -1 unless ($buildtype eq "solr"); #&& $incremental);
    607     }
    608    
    609     # Now the logic in GLI's CollectionManager.java (processComplete()
    610     # and installCollection()) and Gatherer.configGS3Server().
    611    
    612     # 1. Get library URL
    613    
    614     # For web servers that are external to a Greenstone installation,
    615     # the user can pass in their web server's library URL.
    616     # For web servers included with GS (like tomcat for GS3 and server.exe
    617     # and apache for GS2), we work out the library URL:
    618     if(!$library_url) {
    619     $library_url = &get_library_URL($gs_mode, $library_name); # returns undef if no server is running
    620     }
    621    
    622     # CollectionManager's installCollection phase in GLI
    623     # 2. Ping the library URL, and if it's a persistent server and running, release the collection
    624    
    625     my $is_persistent_server = undef;
    626     if(!$skipactivation && $library_url) { # undef if no valid server URL
    627    
     467sub do_deactivate {
     468    my($is_persistent_server, $library_url, $gs_mode, $site, $qualified_collection) = @_;
     469
     470
    628471    &print_msg("Pinging $library_url\n");       
    629472    if (&ping_library($library_url, $gs_mode, $site)) { # server running
     
    654497        &print_msg("No response to Ping => Taken to mean server is not running\n");
    655498    }
    656    
    657     }
    658    
    659     # 2b. If we're working with a solr collection, then start up the solrserver now.
    660     my $solr_server;
    661     my @corenames = ();
    662     if($buildtype eq "solr") { # start up the jetty server 
    663     my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl
    664     unshift (@INC, "$solr_ext/perllib");
    665     require solrserver;
    666 
    667     # Solr cores are named without taking the collection-group name into account, since solr
    668     # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead
    669     my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection);
    670 
    671     # See solrbuilder.pm to get the indexing levels (document, section) from the collectcfg file
    672     # Used to generate core names from them and remove cores by name
    673     foreach my $level ( @{$collectcfg->{'levels'}} ){
    674         my ($pindex) = $level =~ /^(.)/;
    675         my $indexname = $pindex."idx";
    676         push(@corenames, "$site-$colname-$indexname"); #"$site-$colname-didx", "$site-$colname-sidx"
    677         }
    678    
    679     # If the Solr/Jetty server is not already running, the following starts
    680     # it up, and only returns when the server is "reading and listening"   
    681     $solr_server = new solrserver($build_dir);
    682     $solr_server->start();
    683    
    684     # We'll be moving building to index. For solr collection, there's further
    685     # special processing to make a corresponding change to the solr.xml
    686     # by removing the temporary building cores and (re)creating the index cores
    687     }
    688 
    689 
    690     # 3. Do all the moving building to index stuff now 
    691    
    692     # If removeold: replace index dir with building dir.
    693     # If keepold: move building's contents into index, where only duplicates will get deleted.
    694     # removeold and keepold can't both be on at the same time
    695     # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does (moving building to index)
    696     my $incremental_mode;
    697     ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold,
    698                            $incremental,
    699                            $build_dir, # checkdir. Usually archives or export to be deleted. activate.pl deletes building
    700                            $collectcfg);
    701    
    702     if($removeold) {
    703    
    704     if(&FileUtils::directoryExists($index_dir)) {
    705         &print_task_msg("Removing \"index\"");
    706        
    707         if ($buildtype eq "solr") {
    708         # if solr, remove any cores that are using the index_dir before deleting this dir
    709         foreach my $corename (@corenames) {
    710             $solr_server->admin_unload_core($corename);
    711         }
    712         }   
    713        
    714         &FileUtils::removeFilesRecursive($index_dir);
    715        
    716         # Wait for a couple of seconds, just for luck
    717         sleep 2;
    718        
    719         if (&FileUtils::directoryExists($index_dir)) {
    720         &print_msg("The index directory $index_dir could not be deleted.\n"); # CollectionManager.Index_Not_Deleted
    721         }
    722     }
    723    
    724     # if remote GS server: gliserver.pl would call activate.pl to activate
    725     # the collection at this point since activate.pl lives on the server side
    726    
    727     if ($buildtype eq "solr") {
    728         # if solr, remove any cores that are using the building_dir before moving this dir onto index
    729         foreach my $corename (@corenames) {
    730         $solr_server->admin_unload_core("building-$corename");
    731         }
    732     }
    733    
    734     # Move the building directory to become the new index directory
    735     &print_task_msg("Moving \"building\" -> \"index\"");
    736     &FileUtils::moveFiles($build_dir, $index_dir);
    737     if(&FileUtils::directoryExists($build_dir) || !&FileUtils::directoryExists($index_dir)) {           
    738         &print_msg("Could not move $build_dir to $index_dir.\n"); # CollectionManager.Build_Not_Moved
    739     }
    740     }
    741     elsif ($keepold || $incremental) {
    742     if ($buildtype eq "solr") {
    743         # if solr, remove any cores that may be using the building_dir before moving this dir onto index
    744         foreach my $corename (@corenames) {         
    745         $solr_server->admin_unload_core("building-$corename") if $solr_server->admin_ping_core("building-$corename");
    746         }
    747     }
    748    
    749     if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl
    750         &print_task_msg("building folder is index folder, not moving");
    751     } else {
    752         # Copy just the contents of building dir into the index dir, overwriting
    753         # existing files, but don't replace index with building.
    754         &print_task_msg("Moving \"building\" -> \"index\"");       
    755         &FileUtils::moveDirectoryContents($build_dir, $index_dir);
    756     }
    757     }
    758    
    759     if ($buildtype eq "solr") {
    760     # Call CREATE action to get the old cores pointing to the index folder
    761     foreach my $corename (@corenames) {
    762         if($removeold) {
    763         # Call CREATE action to get all cores pointing to the index folder, since building is now index
    764         $solr_server->admin_create_core($corename, $index_dir);
    765        
    766         } elsif ($keepold || $incremental) {
    767         # Call RELOAD core. Should already be using the index_dir directory for $keepold and $incremental case
    768        
    769         # Ping to see if corename exists, if it does, reload, else create
    770         if ($solr_server->admin_ping_core($corename)) {
    771             $solr_server->admin_reload_core($corename);
    772         } else {
    773             $solr_server->admin_create_core($corename, $index_dir);
    774         }
    775         }
    776     }
    777    
    778     # regenerate the solr.xml.in from solr.xml in case we are working off a dvd.
    779     $solr_server->solr_xml_to_solr_xml_in();
    780     }
    781    
    782     # 4. Ping the library URL, and if it's a persistent server and running, activate the collection again   
    783    
    784     # Check for success: if building does not exist OR is empty OR if building is index (in which case there was no move)   
    785     if($build_dir eq $index_dir || !&FileUtils::directoryExists($build_dir) || &FileUtils::isDirectoryEmpty($build_dir)) {
    786    
    787     if(!$skipactivation && $library_url) { # undef if no valid server URL
    788        
     499
     500    return $is_persistent_server;
     501}
     502
     503sub do_activate {
     504    my($is_persistent_server, $library_url, $gs_mode, $site, $qualified_collection) = @_;
     505
     506
    789507        &print_msg("Pinging $library_url\n");
    790508        if (&ping_library($library_url, $gs_mode, $site)) { # server running
     
    830548        &print_msg("No response to Ping => Taken to mean server is not running\n");
    831549        }
     550
     551    return $is_persistent_server;
     552}
     553
     554
     555# Most of the arguments are familiar from the building scripts like buildcol.pl
     556# The special optional argument -library_url is for when we're dealing with a web
     557# library server such as an apache that's separate from any included with GS2.
     558# In such a case, this script's caller should pass in -library_url <URL>.
     559#
     560# $site argument must be specified in the cmdline for collectionConfig.xml to get
     561# generated which makes $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
     562sub main
     563{
     564    my ($argc,@argv) = @_;
     565
     566    if (($argc==0)  || (($argc==1) && ($argv[0] =~ m/^--?h(elp)?$/))) {
     567    my ($progname) = ($0 =~ m/^.*[\/|\\](.*?)$/);
     568   
     569   
     570    print STDERR "\n";
     571    print STDERR "Usage: $progname [-collectdir c -builddir b -indexdir i -site s -skipactivation -removeold -keepold -verbosity v\n";
     572    print STDERR "\t-library_url URL -library_name n] <[colgroup/]collection>\n";
     573    print STDERR "\n";
     574   
     575    exit(-1);
     576    }
     577   
     578    # http://stackoverflow.com/questions/6156742/how-can-i-capture-the-complete-commandline-in-perl
     579    #print STDERR "@@@@@@@@@ ACTIVATE CMD: " . join " ", $0, @ARGV;
     580   
     581    # get the collection details
     582    my $qualified_collection = pop @argv; # qualified collection
     583   
     584    my $collect_dir = undef; #"collect"; # can't be "collect" when only -site is provided for GS3
     585    my $build_dir = undef;
     586    my $index_dir = undef;
     587    my $site = undef;
     588   
     589    # if run from server (java code), it will handle deactivation and activation to prevent open file handles when java launches this script and exits:
     590    my $skipactivation = 0;
     591    my $removeold = 0;
     592    my $keepold = 0;
     593    my $incremental = 0; # used by solr
     594   
     595    # some further parameters
     596    my $just_activate = 0; # don't do anything else than activating the collection if the server is running
     597    my $just_deactivate = 0; # don't do anything else than deactivating the collection if the server is running
     598
     599
     600    my $library_url = $ENV{'GSDL_LIBRARY_URL'} || undef; # to be specified on the cmdline if not using a GS-included web server
     601    # the GSDL_LIBRARY_URL env var is useful when running cmdline buildcol.pl in the linux package manager versions of GS3
     602   
     603    my $library_name = undef;
     604   
     605    while (my $arg = shift @argv) {
     606    if ($arg eq "-collectdir") {
     607        $collect_dir = shift @argv;
     608    }
     609    elsif ($arg eq "-builddir") {
     610        $build_dir = shift @argv;
     611    }
     612    elsif ($arg eq "-indexdir") {
     613        $index_dir = shift @argv;
     614    }
     615    elsif ($arg eq "-site") {
     616        $site = shift @argv;
     617    }
     618    elsif ($arg eq "-skipactivation") {
     619        $skipactivation = 1;
     620    }
     621    elsif ($arg eq "-removeold") {
     622        $removeold = 1;
     623    }
     624    elsif ($arg eq "-keepold") {
     625        $keepold = 1;
     626    }
     627    elsif ($arg eq "-incremental") {
     628        $incremental = 1;
     629    }   
     630    elsif ($arg eq "-justactivate") {
     631        $just_activate = 1;
     632    }
     633    elsif ($arg eq "-justdeactivate") {
     634        $just_deactivate = 1;
     635    }
     636    elsif ($arg eq "-library_url") {
     637        $library_url = shift @argv;
     638    }
     639    elsif ($arg eq "-library_name") {
     640        $library_name = shift @argv;
     641    }
     642    elsif ($arg eq "-verbosity") {
     643        $default_verbosity = shift @argv; # global variable
     644       
     645        # ensure we're working with ints not strings (int context not str context), in case verbosity=0
     646        # http://stackoverflow.com/questions/288900/how-can-i-convert-a-string-to-a-number-in-perl
     647        $default_verbosity = int($default_verbosity || 0); ### is this the best way?
     648    }
     649    }
     650   
     651    # work out the building and index dirs
     652    my $collection_dir = &util::resolve_collection_dir($collect_dir, $qualified_collection, $site);
     653    $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir);
     654    $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir);
     655   
     656    &print_task_msg("Running  Collection  Activation  Stage");
     657   
     658    # get and check the collection name
     659    if ((&colcfg::use_collection($site, $qualified_collection, $collect_dir)) eq "") {
     660    &print_msg("Unable to use collection \"$qualified_collection\" within \"$collect_dir\"\n");
     661    exit -1;
     662    }
     663   
     664    # Read in the collection configuration file.
     665    # Beware: Only if $site is specified in the cmdline does collectionConfig.xml get
     666    # generated and does $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
     667    my $gs_mode = "gs2";
     668    if ((defined $site) && ($site ne "")) { # GS3
     669    $gs_mode = "gs3";
     670    }
     671    my $collect_cfg_filename = &colcfg::get_collect_cfg_name(STDERR, $gs_mode);
     672    my $collectcfg = &colcfg::read_collection_cfg ($collect_cfg_filename,$gs_mode);
     673   
     674    # look for build.cfg/buildConfig.xml
     675    my $build_cfg_filename ="";
     676   
     677    if ($gs_mode eq "gs2") {
     678    $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir,"build.cfg");
     679    } else {
     680    $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir, "buildConfig.xml");
     681    # gs_mode is GS3. Set the site now if this was not specified as cmdline argument
     682    #$site = "localsite" unless defined $site;
     683    }
     684   
     685    # We need to know the buildtype for Solr.
     686    # Any change of indexers is already detected and handled by the calling code (buildcol or
     687    # full-rebuild), so that at this stage the config file's buildtype reflects the actual buildtype.
     688   
     689    # From buildcol.pl we use searchtype for determining buildtype, but for old versions, use buildtype
     690    my $buildtype;
     691    if (defined $collectcfg->{'buildtype'}) {
     692    $buildtype = $collectcfg->{'buildtype'};
     693    } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
     694    $buildtype = "mgpp";
     695    } else {
     696    $buildtype = "mg"; #mg is the default
     697    }
     698   
     699    # can't do anything without a build directory with something in it to move into index
     700    # Except if we're (doing incremental) building for solr, where we want to still
     701    # activate and deactivate collections including for the incremental case
     702    if(!&FileUtils::directoryExists($build_dir)) {
     703    &print_msg("No building folder at $build_dir to move to index.\n");
     704    exit -1 unless ($buildtype eq "solr"); #&& $incremental);
     705    } elsif (&FileUtils::isDirectoryEmpty($build_dir)) {
     706    &print_msg("Nothing in building folder $build_dir to move into index folder.\n");
     707    exit -1 unless ($buildtype eq "solr"); #&& $incremental);
     708    }
     709   
     710    # Now the logic in GLI's CollectionManager.java (processComplete()
     711    # and installCollection()) and Gatherer.configGS3Server().
     712   
     713    # 1. Get library URL
     714   
     715    # For web servers that are external to a Greenstone installation,
     716    # the user can pass in their web server's library URL.
     717    # For web servers included with GS (like tomcat for GS3 and server.exe
     718    # and apache for GS2), we work out the library URL:
     719    if(!$library_url) {
     720    $library_url = &get_library_URL($gs_mode, $library_name); # returns undef if no server is running
     721    }
     722   
     723    # CollectionManager's installCollection phase in GLI
     724    # 2. Ping the library URL, and if it's a persistent server and running, release the collection
     725    my $is_persistent_server = undef;
     726    if($library_url && !$skipactivation) { # undef if no valid server URL
     727    if($just_deactivate || !$just_activate) { # either $justdeactivate is provided or neither $justactivate and $justdeactivate are provided
     728        # and deactivation will go through
     729        $is_persistent_server = &do_deactivate($is_persistent_server, $library_url, $gs_mode, $site, $qualified_collection);
     730    }
     731    }
     732
     733    # 2b. If we're working with a solr collection, then start up the solrserver now.
     734    my $solr_server;
     735if(!$just_activate && !$just_deactivate) {
     736    my @corenames = ();
     737    if($buildtype eq "solr") { # start up the jetty server 
     738    my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl
     739    unshift (@INC, "$solr_ext/perllib");
     740    require solrserver;
     741
     742    # Solr cores are named without taking the collection-group name into account, since solr
     743    # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead
     744    my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection);
     745
     746    # See solrbuilder.pm to get the indexing levels (document, section) from the collectcfg file
     747    # Used to generate core names from them and remove cores by name
     748    foreach my $level ( @{$collectcfg->{'levels'}} ){
     749        my ($pindex) = $level =~ /^(.)/;
     750        my $indexname = $pindex."idx";
     751        push(@corenames, "$site-$colname-$indexname"); #"$site-$colname-didx", "$site-$colname-sidx"
     752        }
     753   
     754    # If the Solr/Jetty server is not already running, the following starts
     755    # it up, and only returns when the server is "reading and listening"   
     756    $solr_server = new solrserver($build_dir);
     757    $solr_server->start();
     758   
     759    # We'll be moving building to index. For solr collection, there's further
     760    # special processing to make a corresponding change to the solr.xml
     761    # by removing the temporary building cores and (re)creating the index cores
     762    }
     763
     764
     765    # 3. Do all the moving building to index stuff now 
     766   
     767    # If removeold: replace index dir with building dir.
     768    # If keepold: move building's contents into index, where only duplicates will get deleted.
     769    # removeold and keepold can't both be on at the same time
     770    # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does (moving building to index)
     771    my $incremental_mode;
     772    ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold,
     773                           $incremental,
     774                           $build_dir, # checkdir. Usually archives or export to be deleted. activate.pl deletes building
     775                           $collectcfg);
     776   
     777    if($removeold) {
     778   
     779    if(&FileUtils::directoryExists($index_dir)) {
     780        &print_task_msg("Removing \"index\"");
     781       
     782        if ($buildtype eq "solr") {
     783        # if solr, remove any cores that are using the index_dir before deleting this dir
     784        foreach my $corename (@corenames) {
     785            $solr_server->admin_unload_core($corename);
     786        }
     787        }   
     788       
     789        &FileUtils::removeFilesRecursive($index_dir);
     790       
     791        # Wait for a couple of seconds, just for luck
     792        sleep 2;
     793       
     794        if (&FileUtils::directoryExists($index_dir)) {
     795        &print_msg("The index directory $index_dir could not be deleted.\n"); # CollectionManager.Index_Not_Deleted
     796        }
     797    }
     798   
     799    # if remote GS server: gliserver.pl would call activate.pl to activate
     800    # the collection at this point since activate.pl lives on the server side
     801   
     802    if ($buildtype eq "solr") {
     803        # if solr, remove any cores that are using the building_dir before moving this dir onto index
     804        foreach my $corename (@corenames) {
     805        $solr_server->admin_unload_core("building-$corename");
     806        }
     807    }
     808   
     809    # Move the building directory to become the new index directory
     810    &print_task_msg("Moving \"building\" -> \"index\"");
     811    &FileUtils::moveFiles($build_dir, $index_dir);
     812    if(&FileUtils::directoryExists($build_dir) || !&FileUtils::directoryExists($index_dir)) {           
     813        &print_msg("Could not move $build_dir to $index_dir.\n"); # CollectionManager.Build_Not_Moved
     814    }
     815    }
     816    elsif ($keepold || $incremental) {
     817    if ($buildtype eq "solr") {
     818        # if solr, remove any cores that may be using the building_dir before moving this dir onto index
     819        foreach my $corename (@corenames) {         
     820        $solr_server->admin_unload_core("building-$corename") if $solr_server->admin_ping_core("building-$corename");
     821        }
     822    }
     823   
     824    if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl
     825        &print_task_msg("building folder is index folder, not moving");
     826    } else {
     827        # Copy just the contents of building dir into the index dir, overwriting
     828        # existing files, but don't replace index with building.
     829        &print_task_msg("Moving \"building\" -> \"index\"");       
     830        &FileUtils::moveDirectoryContents($build_dir, $index_dir);
     831    }
     832    }
     833   
     834    if ($buildtype eq "solr") {
     835    # Call CREATE action to get the old cores pointing to the index folder
     836    foreach my $corename (@corenames) {
     837        if($removeold) {
     838        # Call CREATE action to get all cores pointing to the index folder, since building is now index
     839        $solr_server->admin_create_core($corename, $index_dir);
     840       
     841        } elsif ($keepold || $incremental) {
     842        # Call RELOAD core. Should already be using the index_dir directory for $keepold and $incremental case
     843       
     844        # Ping to see if corename exists, if it does, reload, else create
     845        if ($solr_server->admin_ping_core($corename)) {
     846            $solr_server->admin_reload_core($corename);
     847        } else {
     848            $solr_server->admin_create_core($corename, $index_dir);
     849        }
     850        }
     851    }
     852   
     853    # regenerate the solr.xml.in from solr.xml in case we are working off a dvd.
     854    $solr_server->solr_xml_to_solr_xml_in();
     855    }
     856}#end if(!$just_activate && !$just_deactivate)
     857   
     858    # 4. Ping the library URL, and if it's a persistent server and running, activate the collection again   
     859   
     860    # Check for success: if building does not exist OR is empty OR if building is index (in which case there was no move)   
     861    if($build_dir eq $index_dir || !&FileUtils::directoryExists($build_dir) || &FileUtils::isDirectoryEmpty($build_dir)) {
     862   
     863    if($library_url && (!$skipactivation)) { # undef if no valid server URL
     864        if($just_activate || !$just_deactivate) { # either $justactivate is provided or neither $justdeactivate and $justactivate are provided
     865        # then activation will go through       
     866        $is_persistent_server = &do_activate($is_persistent_server, $library_url, $gs_mode, $site, $qualified_collection);
     867        }   
    832868    }
    833869    } else { # installcollection failed     
     
    838874    &print_msg("\n");
    839875   
    840     if($buildtype eq "solr") {
     876    if(!$just_activate && !$just_deactivate && $buildtype eq "solr") {
    841877    if ($solr_server->explicitly_started()) {
    842878        $solr_server->stop();
  • main/trunk/greenstone2/perllib/DBDrivers/BaseDBDriver.pm

    r30370 r30517  
    5757    $self->{'supports_persistentconnection'} = 0;
    5858    $self->{'supports_rss'} = 0;
     59    $self->{'supports_concurrent_read_and_write'} = 0;
    5960    $self->{'supports_set'} = 0;
    6061    $self->{'write_only'} = 0; # Some drivers are one way - i.e. STDOUTXML
     
    383384
    384385
     386## @function supportsConcurrentReadAndWrite(void)  => integer
     387#
     388sub supportsConcurrentReadAndWrite
     389{
     390    my $self = shift(@_);
     391    return $self->{'supports_concurrent_read_and_write'};
     392}
     393## supportsConcurrentReadAndWrite(void) => integer ##
     394
     395
    385396## @function supportsSet(void) => integer
    386397#
  • main/trunk/greenstone2/perllib/DBDrivers/JDBM.pm

    r30370 r30517  
    5050    my $self = DBDrivers::70HyphenFormat->new(@_);
    5151    $self->{'default_file_extension'} = 'jdb';
     52    $self->{'supports_concurrent_read_and_write'} = 1;
    5253
    5354    # Executables need a little extra work since we are using Java
  • main/trunk/greenstone2/perllib/basebuilder.pm

    r28567 r30517  
    397397}
    398398
     399# By default, builders do support make_infodatabase()
     400sub supports_make_infodatabase {
     401    return 1;
     402}
    399403
    400404
  • main/trunk/greenstone2/perllib/buildcolutils.pm

    r29096 r30517  
    674674    #   processor [jmt12]
    675675    map { local $_=$_; $_->build_indexes($indexname, $indexlevel); } @builders;
    676     map { local $_=$_; $_->make_infodatabase(); }  @builders;
     676
     677    # when incrementally rebuilding a collection using a gdbm database, need to
     678    # deactivate the collecitob before make_infodatabase()
     679    map {
     680    local $_=$_;
     681
     682    if($_->supports_make_infodatabase()) {
     683        my $infodbtype = $_->{'infodbtype'};
     684        my $dbSupportsConcurrentRW = &dbutil::supportsConcurrentReadAndWrite($infodbtype);
     685   
     686        if(!$dbSupportsConcurrentRW) {
     687        #$self->{'justdeactivate'} = 1;
     688        #$self->activate_collection();
     689        #$self->{'justdeactivate'} = 0;
     690        $self->activate_collection("just_deactivate");     
     691        }
     692        $_->make_infodatabase();       
     693    }
     694
     695    }  @builders;
     696
    677697    map { local $_=$_; $_->collect_specific(); } @builders;
    678698  }
     
    759779sub activate_collection
    760780{
    761   my $self = shift(@_);
     781  my $self = shift(@_); 
     782  my $activation_cmd = shift(@_);
     783  my($just_activate, $just_deactivate);
     784  if(defined $activation_cmd) {
     785      $just_deactivate = 1 if($activation_cmd eq "just_deactivate");
     786      $just_activate = 1 if($activation_cmd eq "just_activate");
     787      if($activation_cmd eq "just_deactivate_and_activate") {
     788      $just_deactivate = 1;
     789      $just_activate = 1;
     790      }
     791  }
     792
    762793  # if buildcol.pl was run with -activate, need to run activate.pl
    763794  # now that building's complete
     
    773804    push(@activate_argv, '-keepold') if ($self->{'keepold'});
    774805    push(@activate_argv, '-incremental') if ($self->{'incremental'});
     806    push(@activate_argv, '-justactivate') if ($just_activate || $self->{'justactivate'});
     807    push(@activate_argv, '-justdeactivate') if ($just_deactivate || $self->{'justdeactivate'});
    775808    my $quoted_argv = join(' ', map { "\"$_\"" } @activate_argv);
    776809    my $activatecol_cmd = '"' . &util::get_perl_exec(). '" -S activate.pl ' . $quoted_argv . ' "' . $self->get_collection() . '"';
  • main/trunk/greenstone2/perllib/dbutil.pm

    r30489 r30517  
    532532
    533533
     534## @function supportsConcurrentReadAndWrite(string)  => boolean
     535#
     536sub supportsConcurrentReadAndWrite
     537{
     538    my $infodb_type = shift(@_);
     539    my $driver = _loadDBDriver($infodb_type);
     540    return $driver->supportsConcurrentReadAndWrite();
     541}
     542## supportsConcurrentReadAndWrite(string) => boolean ##
     543
     544
    534545## @function write_infodb_entry(string, *) => void
    535546#
  • main/trunk/greenstone2/perllib/extrabuilder.pm

    r24462 r30517  
    113113}
    114114
     115# for now, orthogonalbuilder subclasses don't support/require make_infodatabase()
     116sub supports_make_infodatabase {
     117    return 0;
     118}
     119
    115120sub make_infodatabase {
    116121    my $self = shift (@_);
  • main/trunk/greenstone2/perllib/inexport.pm

    r30489 r30517  
    12841284        if ($existing_f =~ m/^$situated_dir/) {
    12851285
    1286         print STDERR "**** Existing file $existing_f\nis located within\n$situated_dir\n";
     1286#       print STDERR "**** Existing file $existing_f\nis located within\n$situated_dir\n";
    12871287
    12881288        push(@$reindex_files,$existing_f);
Note: See TracChangeset for help on using the changeset viewer.