source: main/trunk/greenstone2/bin/script/activate.pl

Last change on this file was 38509, checked in by davidb, 4 months ago

print_task_msg already includes an newline char

  • Property svn:executable set to *
File size: 16.9 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# activate.pl -- to be called after building a collection to activate it.
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 2009 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29
30# This program is designed to support the building process of Greenstone.
31# It deactivates the collection just built, if the web server is running
32# and is a persistent web server (or if the library_URL provided as
33# parameter to this script is of a currently running web server). It then
34# moves building to index, before activating the collection on the GS2 or
35# GS3 web server again if necessary.
36
37use Config;
38
39BEGIN {
40 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
41 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
42 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
43 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
44
45 # Adding cpan in, adds in its auto subfolder which conflicts with ActivePerl on Windows
46 # The auto folder has been moved into a perl-5.8 folder, and this will now be included
47 # only if the current version of perl is 5.8 (and not ActivePerl).
48 my $perl_dir;
49
50 # Note: $] encodes the version number of perl
51 if ($]>=5.010) {
52 $perl_dir="perl-5.".substr($],3,2);
53 }
54 elsif ($]>5.008) {
55 # perl 5.8.1 or above
56 $perl_dir = "perl-5.8";
57 }
58 elsif ($]>=5.008) {
59 # perl 5.8.1 or above
60 $perl_dir = "perl-5.8";
61 }
62 elsif ($]<5.008) {
63 # assume perl 5.6
64 $perl_dir = "perl-5.6";
65 }
66 else {
67 print STDERR "Warning: Perl 5.8.0 is not a maintained release.\n";
68 print STDERR " Please upgrade to a newer version of Perl.\n";
69 $perl_dir = "perl-5.8";
70 }
71
72 #if ($ENV{'GSDLOS'} !~ /^windows$/i) {
73 # Use push to put this on the end, so an existing XML::Parser will be used by default
74 if (-d "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir-mt" && $Config{usethreads}){
75 push (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir-mt");
76 }
77 else{
78 push (@INC, "$ENV{'GSDLHOME'}/perllib/cpan/$perl_dir");
79 }
80 #}
81
82}
83
84
85use strict;
86no strict 'refs'; # allow filehandles to be variables and vice versa
87no strict 'subs'; # allow barewords (eg STDERR) as function arguments
88
89use File::Basename;
90use File::Find;
91
92# Greenstone modules
93use colcfg;
94use oaiinfo;
95use scriptutil;
96use servercontrol;
97use util;
98
99
100# Most of the arguments are familiar from the building scripts like buildcol.pl
101# The special optional argument -library_url is for when we're dealing with a web
102# library server such as an apache that's separate from any included with GS2.
103# In such a case, this script's caller should pass in -library_url <URL>.
104#
105# $site argument must be specified in the cmdline for collectionConfig.xml to get
106# generated which makes $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
107sub main
108{
109 my ($argc,@argv) = @_;
110
111 if (($argc==0) || (($argc==1) && ($argv[0] =~ m/^--?h(elp)?$/))) {
112 my ($progname) = ($0 =~ m/^.*[\/|\\](.*?)$/);
113
114
115 print STDERR "\n";
116 print STDERR "Usage: $progname [-collectdir c -builddir b -indexdir i -site s -skipactivation -removeold -keepold -incremental -verbosity v\n";
117 print STDERR "\t-library_url URL -library_name n] <[colgroup/]collection>\n";
118 print STDERR "\n";
119
120 exit(-1);
121 }
122
123 # http://stackoverflow.com/questions/6156742/how-can-i-capture-the-complete-commandline-in-perl
124 #print STDERR "@@@@@@@@@ ACTIVATE CMD: " . join " ", $0, @ARGV . "\n";
125
126 # get the collection details
127 my $qualified_collection = pop @argv; # qualified collection
128
129 my $collect_dir = undef; #"collect"; # can't be "collect" when only -site is provided for GS3
130 my $build_dir = undef;
131 my $index_dir = undef;
132 my $site = undef;
133
134 # if run from server (java code), it will handle deactivation and activation to prevent open file handles when java launches this script and exits:
135 my $skipactivation = 0;
136 my $removeold = 0;
137 my $keepold = 0;
138 my $incremental = 0; # used by solr
139
140 my $default_verbosity = 2;
141
142 my $library_url = undef; # to be specified on the cmdline if not using a GS-included web server
143 # the GSDL_LIBRARY_URL env var is useful when running cmdline buildcol.pl in the linux package manager versions of GS3
144
145 my $library_name = undef;
146 my $oaiserver_name = undef;
147
148 while (my $arg = shift @argv) {
149 if ($arg eq "-collectdir") {
150 $collect_dir = shift @argv;
151 }
152 elsif ($arg eq "-builddir") {
153 $build_dir = shift @argv;
154 }
155 elsif ($arg eq "-indexdir") {
156 $index_dir = shift @argv;
157 }
158 elsif ($arg eq "-site") {
159 $site = shift @argv;
160 }
161 elsif ($arg eq "-skipactivation") {
162 $skipactivation = 1;
163 }
164 elsif ($arg eq "-removeold") {
165 $removeold = 1;
166 }
167 elsif ($arg eq "-keepold") {
168 $keepold = 1;
169 }
170 elsif ($arg eq "-incremental") {
171 $incremental = 1;
172 }
173 elsif ($arg eq "-library_url") {
174 $library_url = shift @argv;
175 }
176 elsif ($arg eq "-library_name") {
177 $library_name = shift @argv;
178 }
179 elsif ($arg eq "-oaiserver_name") {
180 $oaiserver_name = shift @argv;
181 }
182 elsif ($arg eq "-verbosity") {
183 $default_verbosity = shift @argv; # global variable
184
185 # ensure we're working with ints not strings (int context not str context), in case verbosity=0
186 # http://stackoverflow.com/questions/288900/how-can-i-convert-a-string-to-a-number-in-perl
187 $default_verbosity = int($default_verbosity || 0); ### is this the best way?
188 }
189 }
190
191 ## shortcut - if gsdl3srchome is set, and site is not set, set site to localsite
192 ## But need to check gsdl3srchome not empty even if defined, as that can
193 ## happen when nightly diffcol is run and doesn't mean it's a GS3 install
194 if (defined $ENV{'GSDL3SRCHOME'} && $ENV{'GSDL3SRCHOME'} ne "") {
195 if (!defined $site || $site eq "") {
196 $site = "localsite";
197 print STDOUT "setting site to localsite as GSDL3SRCHOME was set\n" if $default_verbosity > 3;
198 }
199 }
200 # work out the building and index dirs
201 my $collection_dir = &util::resolve_collection_dir($collect_dir, $qualified_collection, $site);
202 $build_dir = &FileUtils::filenameConcatenate($collection_dir, "building") unless (defined $build_dir);
203 $index_dir = &FileUtils::filenameConcatenate($collection_dir, "index") unless (defined $index_dir);
204 my $solr_conf_dir = &FileUtils::filenameConcatenate($collection_dir, "etc", "conf");
205
206 my $gsserver = new servercontrol($qualified_collection, $site, $default_verbosity, $build_dir, $index_dir, $collect_dir, $library_url, $library_name, $oaiserver_name);
207
208 $gsserver->print_task_msg("Running Collection Activation Stage");
209
210 # get and check the collection name
211 if ((&colcfg::use_collection($site, $qualified_collection, $collect_dir)) eq "") {
212 $gsserver->print_msg("Unable to use collection \"$qualified_collection\" within \"$collect_dir\"\n");
213 exit -1;
214 }
215
216 # Read in the collection configuration file.
217 # Beware: Only if $site is specified in the cmdline does collectionConfig.xml get
218 # generated and does $gs_mode=gs3, else collect.cfg gets generated and $gs_mode=gs2
219 my $gs_mode = $gsserver->{'gs_mode'}; # "gs2" or "gs3", based on $site variable
220
221 my $collect_cfg_filename = &colcfg::get_collect_cfg_name(STDERR, $gs_mode);
222 my $collectcfg = &colcfg::read_collection_cfg ($collect_cfg_filename,$gs_mode);
223
224 # look for build.cfg/buildConfig.xml
225 my $build_cfg_filename ="";
226
227 if ($gs_mode eq "gs2") {
228 $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir,"build.cfg");
229 } else {
230 $build_cfg_filename = &FileUtils::filenameConcatenate($build_dir, "buildConfig.xml");
231 # gs_mode is GS3. Set the site now if this was not specified as cmdline argument
232 #$site = "localsite" unless defined $site;
233 }
234
235 my $buildcfg = &colcfg::read_building_cfg($build_cfg_filename, $gs_mode);
236
237 # We need to know the buildtype for Solr.
238 # Any change of indexers is already detected and handled by the calling code (buildcol or
239 # full-rebuild), so that at this stage the config file's buildtype reflects the actual buildtype.
240
241 # From buildcol.pl we use searchtype for determining buildtype, but for old versions, use buildtype
242 my $buildtype;
243 if (defined $collectcfg->{'buildtype'}) {
244 $buildtype = $collectcfg->{'buildtype'};
245 } elsif (defined $collectcfg->{'searchtypes'} || defined $collectcfg->{'searchtype'}) {
246 $buildtype = "mgpp";
247 } else {
248 $buildtype = "mg"; #mg is the default
249 }
250
251 # can't do anything without a build directory with something in it to move into index
252 # Except if we're (doing incremental) building for solr, where we want to still
253 # activate and deactivate collections including for the incremental case
254
255 if(!$incremental) { # if (!($incremental && ($build_dir eq $index_dir)))
256
257 if(!&FileUtils::directoryExists($build_dir)) {
258 $gsserver->print_msg("No building folder at $build_dir to move to index.\n");
259 exit -1 unless ($buildtype eq "solr"); #&& $incremental);
260 } elsif (&FileUtils::isDirectoryEmpty($build_dir)) {
261 $gsserver->print_msg("Nothing in building folder $build_dir to move into index folder.\n");
262 exit -1 unless ($buildtype eq "solr"); #&& $incremental);
263 }
264 }
265
266 # Now the logic in GLI's CollectionManager.java (processComplete()
267 # and installCollection()) and Gatherer.configGS3Server().
268
269 # 1. Get library URL
270 # CollectionManager's installCollection phase in GLI:
271 # 2. Ping the library URL, and if it's a persistent server and running, release the collection
272 $gsserver->do_deactivate() unless $skipactivation;
273
274 # solr core reloading - previously for incremental building, the solr core is unloaded and created
275 # - why is this? Does this actually need to happen? The following assumes that we *do* want to
276 # unload and create the core even if incremental building
277
278 # solr deactivation. Are we a solr collection? unload any cores that are
279 # present for this collection
280 # If we are not a solr collection, is there etc/solr/conf?
281 # if so, we were a solr collection previously, need to unload cores.
282 # in both cases, we attempt to unload didx and sidx cores, as well
283 # as both building cores
284
285 my $collection_was_previously_solr = 0;
286 if ($buildtype ne "solr" && FileUtils::directoryExists($solr_conf_dir)) {
287 $collection_was_previously_solr = 1;
288 }
289
290 # 2b. If we're working with a solr collection, then start up the solrserver now and unload
291 # any cores that may exist for this collection (building and index cores)
292 my $solr_server;
293 my @corenames = ();
294 my @all_cores = ();
295 my $core_basename = "";
296
297 # for unloading, lets try unloading both didx and sidx as these may have
298 # both been there previously, and if the user removed one in the config file
299 # we won't know that.
300 if($buildtype eq "solr" || $collection_was_previously_solr) {
301
302 my $solr_ext = $ENV{'GEXT_SOLR'}; # from solr_passes.pl
303 unshift (@INC, "$solr_ext/perllib");
304 require solrserver;
305
306 # Solr cores are named without taking the collection-group name into account, since solr
307 # is used for GS3 and GS3 doesn't use collection groups but has the site concept instead
308 my ($colname, $colgroup) = &util::get_collection_parts($qualified_collection);
309 $core_basename = "$site-$colname";
310
311 # If the Solr server is not already running, the following starts
312 # it up, and only returns when the server is "ready and listening"
313 $solr_server = new solrserver($build_dir);
314 $solr_server->start();
315
316 $gsserver->print_task_msg("unloading all solr cores associated with $core_basename & building-$core_basename");
317 $solr_server->admin_unload_all_cores_for_prefix($core_basename);
318 $solr_server->admin_unload_all_cores_for_prefix("building-$core_basename");
319
320 # tidy up other files if we were solr but are not anymore
321 if ($collection_was_previously_solr) {
322 # regenerate the solr.xml.in file to reflect the changes to cores
323 $solr_server->solr_xml_to_solr_xml_in();
324 # we are finished with solrserver now, stop it if necessary
325 if ($solr_server->explicitly_started()) {
326 $solr_server->stop();
327 }
328 # lets delete the conf folder as we are no longer solr
329 &FileUtils::removeFilesRecursive($solr_conf_dir);
330 }
331
332 }
333 # 3. Do all the moving building to index stuff now
334
335 # If removeold: replace index dir with building dir.
336 # If keepold: move building's contents into index, where only duplicates will get deleted.
337 # removeold and keepold can't both be on at the same time
338 # incremental becomes relevant for solr, though it was irrelevant to what activate.pl does
339 # (moving building to index)
340 my $incremental_mode;
341 ($removeold, $keepold, $incremental, $incremental_mode) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, $incremental, $index_dir, $collectcfg);
342
343 if($removeold) {
344
345 if(&FileUtils::directoryExists($index_dir)) {
346 $gsserver->print_task_msg("Removing \"index\"");
347
348 &FileUtils::removeFilesRecursive($index_dir);
349
350 # Wait for a couple of seconds, just for luck
351 sleep 2;
352
353 if (&FileUtils::directoryExists($index_dir)) {
354 $gsserver->print_msg("The index directory $index_dir could not be deleted.\n"); # CollectionManager.Index_Not_Deleted
355 exit -1;
356 }
357 }
358
359 # Move the building directory to become the new index directory
360 $gsserver->print_task_msg("Moving \"building\" -> \"index\"");
361 &FileUtils::moveFiles($build_dir, $index_dir);
362 if(&FileUtils::directoryExists($build_dir) || !&FileUtils::directoryExists($index_dir)) {
363 $gsserver->print_msg("Could not move $build_dir to $index_dir.\n"); # CollectionManager.Build_Not_Moved
364 exit -1;
365 }
366 }
367
368 elsif ($keepold || $incremental) {
369
370 if($build_dir eq $index_dir) { # building_dir can have been set to "index" folder, see incremental-buildcol.pl
371 $gsserver->print_task_msg("building folder is index folder, not moving");
372 } else {
373 # Copy just the contents of building dir into the index dir, overwriting
374 # existing files, but don't replace index with building.
375 $gsserver->print_task_msg("Moving \"building\" -> \"index\"");
376 &FileUtils::moveDirectoryContents($build_dir, $index_dir);
377 }
378 }
379
380
381 # now we've moved building to index, move tmp oaidb to live oaidb in parallel
382 my $oai_info = new oaiinfo($collect_cfg_filename, $collectcfg->{'infodbtype'}, $default_verbosity);
383 $oai_info->activate_collection();
384
385
386 if ($buildtype eq "solr") {
387 # Call CREATE action to get the old cores pointing to the index folder
388 # -- any building or index cores have been unloaded already
389 # -- load up the new one
390
391 my @corenames = @{$buildcfg->{'solrcores'}};
392
393 foreach my $corename (@corenames) {
394 $gsserver->print_task_msg("activating core $core_basename-$corename");
395 $solr_server->admin_create_core("$core_basename-$corename", $index_dir);
396 }
397
398 # regenerate the solr.xml.in from solr.xml in case we are working off a dvd.
399 $solr_server->solr_xml_to_solr_xml_in();
400 # stop the server if necessary
401 if ($solr_server->explicitly_started()) {
402 $solr_server->stop();
403 }
404
405 }
406
407 # 4. Ping the library URL, and if it's a persistent server and running, activate the collection again
408
409 # Check for success: if building does not exist OR is empty OR if building is index (in which case there was no move)
410 if($build_dir eq $index_dir || !&FileUtils::directoryExists($build_dir) || &FileUtils::isDirectoryEmpty($build_dir)) {
411
412 $gsserver->do_activate() unless $skipactivation;
413
414 } else { # installcollection failed
415 #CollectionManager.Preview_Ready_Failed
416 $gsserver->print_msg("Building directory is not empty or still exists. Failed to properly move $build_dir to $index_dir.\n");
417 }
418
419 $gsserver->print_msg("\n");
420
421}
422
423&main(scalar(@ARGV),@ARGV);
Note: See TracBrowser for help on using the repository browser.