root/main/trunk/greenstone2/bin/script/activate.pl @ 30520

Revision 30520, 16.2 KB (checked in by ak19, 4 years ago)

Refactoring activate.pl into activate.pm (class, OOP) and activate.pl. Now buildcolutils.pm uses do_deactivate() from activate.pm.

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# activate.pl -- to be called after building a collection to activate it.
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2009 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30# This program is designed to support the building process of Greenstone.
31# It deactivates the collection just built, if the web server is running
32# and is a persistent web server (or if the library_URL provided as
33# parameter to this script is of a currently running web server). It then
34# moves building to index, before activating the collection on the GS2 or
35# GS3 web server again if necessary.
36
37use Config;
38
39BEGIN {
40    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
41    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
42    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
43    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
44   
45    # Adding cpan in, adds in its auto subfolder which conflicts with ActivePerl on Windows
46    # The auto folder has been moved into a perl-5.8 folder, and this will now be included
47    # only if the current version of perl is 5.8 (and not ActivePerl).
48    my $perl_dir;
49
50    # Note: $] encodes the version number of perl
51    if ($]>=5.010) {
52        $perl_dir="perl-5.".substr($],3,2);
53    }
54    elsif ($]>5.008) {
55    # perl 5.8.1 or above
56    $perl_dir = "perl-5.8";
57    }
58    elsif ($]>=5.008) {
59    # perl 5.8.1 or above
60    $perl_dir = "perl-5.8";
61    }
62    elsif ($]<5.008) {
63    # assume perl 5.6
64    $perl_dir = "perl-5.6";
65    }
66    else {
67    print STDERR "Warning: Perl 5.8.0 is not a maintained release.\n";
68    print STDERR "         Please upgrade to a newer version of Perl.\n";
69    $perl_dir = "perl-5.8";
70    }
71   
72    #if ($ENV{'GSDLOS'} !~ /^windows$/i) {
73    # Use push to put this on the end, so an existing XML::Parser will be used by default
74        if (-d "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir-mt" && $Config{usethreads}){
75            push (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir-mt");
76        }
77        else{
78            push (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir");
79        }
80    #}
81   
82}
83
84
85use strict;
86no strict 'refs'; # allow filehandles to be variables and vice versa
87no strict 'subs'; # allow barewords (eg STDERR) as function arguments
88
89use File::Basename;
90use File::Find;
91
92# Greenstone modules
93use activate;
94use colcfg;
95use scriptutil;
96use util;
97
98
99# Most of the arguments are familiar from the building scripts like buildcol.pl
100# The special optional argument -library_url is for when we're dealing with a web
101# library server such as an apache that's separate from any included with GS2.
102# In such a case, this script's caller should pass in -library_url <URL>.
103#
104# $site argument must be specified in the cmdline for collectionConfig.xml to get
105# generated which makes $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
106sub main
107{
108    my ($argc,@argv) = @_;
109
110    if (($argc==0)  || (($argc==1) && ($argv[0] =~ m/^--?h(elp)?$/))) {
111    my ($progname) = ($0 =~ m/^.*[\/|\\](.*?)$/);
112   
113   
114    print STDERR "\n";
115    print STDERR "Usage: $progname [-collectdir c -builddir b -indexdir i -site s -skipactivation -removeold -keepold -verbosity v\n";
116    print STDERR "\t-library_url URL -library_name n] <[colgroup/]collection>\n";
117    print STDERR "\n";
118   
119    exit(-1);
120    }
121   
122    # http://stackoverflow.com/questions/6156742/how-can-i-capture-the-complete-commandline-in-perl
123    #print STDERR "@@@@@@@@@ ACTIVATE CMD: " . join " ", $0, @ARGV . "\n";
124   
125    # get the collection details
126    my $qualified_collection = pop @argv; # qualified collection
127   
128    my $collect_dir = undef; #"collect"; # can't be "collect" when only -site is provided for GS3
129    my $build_dir = undef;
130    my $index_dir = undef;
131    my $site = undef;
132   
133    # if run from server (java code), it will handle deactivation and activation to prevent open file handles when java launches this script and exits:
134    my $skipactivation = 0;
135    my $removeold = 0;
136    my $keepold = 0;
137    my $incremental = 0; # used by solr
138
139    my $default_verbosity = 2;
140
141    my $library_url = undef; # to be specified on the cmdline if not using a GS-included web server
142    # the GSDL_LIBRARY_URL env var is useful when running cmdline buildcol.pl in the linux package manager versions of GS3
143   
144    my $library_name = undef;
145   
146    while (my $arg = shift @argv) {
147    if ($arg eq "-collectdir") {
148        $collect_dir = shift @argv;
149    }
150    elsif ($arg eq "-builddir") {
151        $build_dir = shift @argv;
152    }
153    elsif ($arg eq "-indexdir") {
154        $index_dir = shift @argv;
155    }
156    elsif ($arg eq "-site") {
157        $site = shift @argv;
158    }
159    elsif ($arg eq "-skipactivation") {
160        $skipactivation = 1;
161    }
162    elsif ($arg eq "-removeold") {
163        $removeold = 1;
164    }
165    elsif ($arg eq "-keepold") {
166        $keepold = 1;
167    }
168    elsif ($arg eq "-incremental") {
169        $incremental = 1;
170    }
171    elsif ($arg eq "-library_url") {
172        $library_url = shift @argv;
173    }
174    elsif ($arg eq "-library_name") {
175        $library_name = shift @argv;
176    }
177    elsif ($arg eq "-verbosity") {
178        $default_verbosity = shift @argv; # global variable
179       
180        # ensure we're working with ints not strings (int context not str context), in case verbosity=0
181        # http://stackoverflow.com/questions/288900/how-can-i-convert-a-string-to-a-number-in-perl
182        $default_verbosity = int($default_verbosity || 0); ### is this the best way?
183    }
184    }
185   
186    # work out the building and index dirs
187    my $collection_dir = &util::resolve_collection_dir($collect_dir, $qualified_collection, $site);
188    $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir);
189    $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir);
190
191    my $gsserver = new activate($qualified_collection, $site, $default_verbosity, $build_dir, $index_dir, $collect_dir, $library_url, $library_name);
192
193    $gsserver->print_task_msg("Running  Collection  Activation  Stage");
194   
195    # get and check the collection name
196    if ((&colcfg::use_collection($site, $qualified_collection, $collect_dir)) eq "") {
197    $gsserver->print_msg("Unable to use collection \"$qualified_collection\" within \"$collect_dir\"\n");
198    exit -1;
199    }
200   
201    # Read in the collection configuration file.
202    # Beware: Only if $site is specified in the cmdline does collectionConfig.xml get
203    # generated and does $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
204    my $gs_mode = $gsserver->{'gs_mode'}; # "gs2" or "gs3", based on $site variable
205
206    my $collect_cfg_filename = &colcfg::get_collect_cfg_name(STDERR, $gs_mode);
207    my $collectcfg = &colcfg::read_collection_cfg ($collect_cfg_filename,$gs_mode);
208   
209    # look for build.cfg/buildConfig.xml
210    my $build_cfg_filename ="";
211   
212    if ($gs_mode eq "gs2") {
213    $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir,"build.cfg");
214    } else {
215    $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir, "buildConfig.xml");
216    # gs_mode is GS3. Set the site now if this was not specified as cmdline argument
217    #$site = "localsite" unless defined $site;
218    }
219   
220    # We need to know the buildtype for Solr.
221    # Any change of indexers is already detected and handled by the calling code (buildcol or
222    # full-rebuild), so that at this stage the config file's buildtype reflects the actual buildtype.
223   
224    # From buildcol.pl we use searchtype for determining buildtype, but for old versions, use buildtype
225    my $buildtype;
226    if (defined $collectcfg->{'buildtype'}) {
227    $buildtype = $collectcfg->{'buildtype'};
228    } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
229    $buildtype = "mgpp";
230    } else {
231    $buildtype = "mg"; #mg is the default
232    }
233   
234    # can't do anything without a build directory with something in it to move into index
235    # Except if we're (doing incremental) building for solr, where we want to still
236    # activate and deactivate collections including for the incremental case
237    if(!&FileUtils::directoryExists($build_dir)) {
238    $gsserver->print_msg("No building folder at $build_dir to move to index.\n");
239    exit -1 unless ($buildtype eq "solr"); #&& $incremental);
240    } elsif (&FileUtils::isDirectoryEmpty($build_dir)) {
241    $gsserver->print_msg("Nothing in building folder $build_dir to move into index folder.\n");
242    exit -1 unless ($buildtype eq "solr"); #&& $incremental);
243    }
244   
245    # Now the logic in GLI's CollectionManager.java (processComplete()
246    # and installCollection()) and Gatherer.configGS3Server().
247   
248    # 1. Get library URL
249   
250    # For web servers that are external to a Greenstone installation,
251    # the user can pass in their web server's library URL.
252    # For web servers included with GS (like tomcat for GS3 and server.exe
253    # and apache for GS2), we work out the library URL:
254    if(!$library_url) {
255    $library_url = $gsserver->get_library_URL(); # returns undef if no server is running
256    }
257   
258    # CollectionManager's installCollection phase in GLI
259    # 2. Ping the library URL, and if it's a persistent server and running, release the collection   
260    if($library_url && !$skipactivation) { # undef if no valid server URL   
261    $gsserver->do_deactivate();
262    }
263
264    # 2b. If we're working with a solr collection, then start up the solrserver now.
265    my $solr_server;
266    my @corenames = ();
267    if($buildtype eq "solr") { # start up the jetty server 
268    my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl
269    unshift (@INC, "$solr_ext/perllib");
270    require solrserver;
271
272    # Solr cores are named without taking the collection-group name into account, since solr
273    # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead
274    my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection);
275
276    # See solrbuilder.pm to get the indexing levels (document, section) from the collectcfg file
277    # Used to generate core names from them and remove cores by name
278    foreach my $level ( @{$collectcfg->{'levels'}} ){
279        my ($pindex) = $level =~ /^(.)/;
280        my $indexname = $pindex."idx";
281        push(@corenames, "$site-$colname-$indexname"); #"$site-$colname-didx", "$site-$colname-sidx"
282        }
283   
284    # If the Solr/Jetty server is not already running, the following starts
285    # it up, and only returns when the server is "reading and listening"   
286    $solr_server = new solrserver($build_dir);
287    $solr_server->start();
288   
289    # We'll be moving building to index. For solr collection, there's further
290    # special processing to make a corresponding change to the solr.xml
291    # by removing the temporary building cores and (re)creating the index cores
292    }
293
294
295    # 3. Do all the moving building to index stuff now 
296   
297    # If removeold: replace index dir with building dir.
298    # If keepold: move building's contents into index, where only duplicates will get deleted.
299    # removeold and keepold can't both be on at the same time
300    # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does (moving building to index)
301    my $incremental_mode;
302    ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold,
303                           $incremental,
304                           $build_dir, # checkdir. Usually archives or export to be deleted. activate.pl deletes building
305                           $collectcfg);
306   
307    if($removeold) {
308   
309    if(&FileUtils::directoryExists($index_dir)) {
310        $gsserver->print_task_msg("Removing \"index\"");
311       
312        if ($buildtype eq "solr") {
313        # if solr, remove any cores that are using the index_dir before deleting this dir
314        foreach my $corename (@corenames) {
315            $solr_server->admin_unload_core($corename);
316        }
317        }   
318       
319        &FileUtils::removeFilesRecursive($index_dir);
320       
321        # Wait for a couple of seconds, just for luck
322        sleep 2;
323       
324        if (&FileUtils::directoryExists($index_dir)) {
325        $gsserver->print_msg("The index directory $index_dir could not be deleted.\n"); # CollectionManager.Index_Not_Deleted
326        }
327    }
328   
329    # if remote GS server: gliserver.pl would call activate.pl to activate
330    # the collection at this point since activate.pl lives on the server side
331   
332    if ($buildtype eq "solr") {
333        # if solr, remove any cores that are using the building_dir before moving this dir onto index
334        foreach my $corename (@corenames) {
335        $solr_server->admin_unload_core("building-$corename");
336        }
337    }
338   
339    # Move the building directory to become the new index directory
340    $gsserver->print_task_msg("Moving \"building\" -> \"index\"");
341    &FileUtils::moveFiles($build_dir, $index_dir);
342    if(&FileUtils::directoryExists($build_dir) || !&FileUtils::directoryExists($index_dir)) {           
343        $gsserver->print_msg("Could not move $build_dir to $index_dir.\n"); # CollectionManager.Build_Not_Moved
344    }
345    }
346    elsif ($keepold || $incremental) {
347    if ($buildtype eq "solr") {
348        # if solr, remove any cores that may be using the building_dir before moving this dir onto index
349        foreach my $corename (@corenames) {         
350        $solr_server->admin_unload_core("building-$corename") if $solr_server->admin_ping_core("building-$corename");
351        }
352    }
353   
354    if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl
355        $gsserver->print_task_msg("building folder is index folder, not moving");
356    } else {
357        # Copy just the contents of building dir into the index dir, overwriting
358        # existing files, but don't replace index with building.
359        $gsserver->print_task_msg("Moving \"building\" -> \"index\"");     
360        &FileUtils::moveDirectoryContents($build_dir, $index_dir);
361    }
362    }
363   
364    if ($buildtype eq "solr") {
365    # Call CREATE action to get the old cores pointing to the index folder
366    foreach my $corename (@corenames) {
367        if($removeold) {
368        # Call CREATE action to get all cores pointing to the index folder, since building is now index
369        $solr_server->admin_create_core($corename, $index_dir);
370       
371        } elsif ($keepold || $incremental) {
372        # Call RELOAD core. Should already be using the index_dir directory for $keepold and $incremental case
373       
374        # Ping to see if corename exists, if it does, reload, else create
375        if ($solr_server->admin_ping_core($corename)) {
376            $solr_server->admin_reload_core($corename);
377        } else {
378            $solr_server->admin_create_core($corename, $index_dir);
379        }
380        }
381    }
382   
383    # regenerate the solr.xml.in from solr.xml in case we are working off a dvd.
384    $solr_server->solr_xml_to_solr_xml_in();
385    }
386
387   
388    # 4. Ping the library URL, and if it's a persistent server and running, activate the collection again   
389   
390    # Check for success: if building does not exist OR is empty OR if building is index (in which case there was no move)   
391    if($build_dir eq $index_dir || !&FileUtils::directoryExists($build_dir) || &FileUtils::isDirectoryEmpty($build_dir)) {
392   
393    if($library_url && !$skipactivation) { # undef if no valid server URL
394        $gsserver->do_activate();
395    }
396    } else { # installcollection failed     
397    #CollectionManager.Preview_Ready_Failed
398    $gsserver->print_msg("Building directory is not empty or still exists. Failed to properly move $build_dir to $index_dir.\n");
399    }
400   
401    $gsserver->print_msg("\n");
402   
403    if($buildtype eq "solr") {
404    if ($solr_server->explicitly_started()) {
405        $solr_server->stop();
406    }
407    }
408}
409
410&main(scalar(@ARGV),@ARGV);
Note: See TracBrowser for help on using the browser.