root/main/trunk/greenstone2/perllib/dbutil.pm @ 30356

Revision 30356, 15.3 KB (checked in by jmt12, 5 years ago)

Complete rewrite of dbutil.pm to (rather than have databases hardcoded) use a dynamic object-oriented loading system.

Line 
1###############################################################################
2#
3# dbutil.pm -- functions to handle using dbdrivers
4#
5# Copyright (c) 2015 New Zealand Digital Library Project
6#
7# A component of the Greenstone digital library software from the New Zealand
8# Digital Library Project at the University of Waikato, New Zealand.
9#
10# This program is free software; you can redistribute it and/or modify it under
11# the terms of the GNU General Public License as published by the Free Software
12# Foundation; either version 2 of the License, or (at your option) any later
13# version.
14#
15# This program is distributed in the hope that it will be useful, but WITHOUT
16# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
18# details.
19#
20# You should have received a copy of the GNU General Public License along with
21# this program; if not, write to the Free Software Foundation, Inc., 675 Mass
22# Ave, Cambridge, MA 02139, USA.
23#
24###############################################################################
25
26package dbutil;
27
28# Pragma
29use strict;
30
31# DEBUGGING: You can enable a DBDriver one at a time to ensure they don't have
32# compilation errors.
33BEGIN
34{
35    if (!defined $ENV{'GSDLHOME'} || !defined $ENV{'GSDLOS'}) {
36        die("Error! Environment not prepared. Have you sourced setup.bash?\n");
37    }
38    # Are we running standalone? In which case the INC won't be correct
39    my $perllib_path = $ENV{'GSDLHOME'} . '/perllib';
40    my $all_inc = join(':', @INC);
41    if ($all_inc !~ /$perllib_path/) {
42    unshift(@INC, $perllib_path);
43    unshift(@INC, $ENV{'GSDLHOME'} . '/ext/parallel-building/perllib');
44    unshift(@INC, $ENV{'GSDLHOME'} . '/ext/tdb/perllib');
45    }
46    ## You can uncomment and name a Driver here to test it compiles
47    #require DBDrivers::TDBCLUSTER;
48    #my $driver = DBDrivers::TDBCLUSTER->new(1);
49}
50
51# Libraries
52use Devel::Peek;
53use Time::HiRes qw ( gettimeofday tv_interval );
54use FileUtils;
55use gsprintf 'gsprintf';
56use util;
57
58# Modulino pattern
59__PACKAGE__->main unless caller;
60
61###############################################################################
62## Private
63###############################################################################
64
65## Display debug messages?
66my $debug = 0; # Set to 1 to display
67
68## Keep track of the driver objects we have initialised
69my $dbdriver_pool = {};
70
71# Testing globals
72my $test_count = 0;
73my $pass_count = 0;
74my $skip_count = 0;
75
76
77## @function _addPathsToINC(void) => void
78#
79# A hopefully unused function to ensure the INC path contains all the available
80# perllib directories (from main, collection, and extensions)
81#
82sub _addPathsToINC
83{
84    &_debugPrint('_addPathsToINC() => ', 0);
85    my @possible_paths;
86    #... the main perllib directory...
87    push(@possible_paths, &FileUtils::filenameConcatenate());
88    #... a collection specific perllib directory...
89    if (defined $ENV{'GSDLCOLLECTDIR'} && $ENV{'GSDLCOLLECTION'}) {
90    push(@possible_paths, &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, 'collect', $ENV{'GSDLCOLLECTION'}, 'perllib'));
91    }
92    #... any registered extension may also have a perllib!
93    if (defined $ENV{'GSDLEXTS'} && defined $ENV{'GSDLHOME'}) {
94    foreach my $gs2_extension (split(/:/, $ENV{'GSDLEXTS'})) {
95        push(@possible_paths, &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, 'ext', $gs2_extension, 'perllib'));
96    }
97    }
98    if (defined $ENV{'GSDL3EXTS'} && defined $ENV{'GSDL3SRCHOME'}) {
99    foreach my $gs3_extension (split(/:/, $ENV{'GSDL3EXTS'})) {
100        push(@possible_paths, &FileUtils::filenameConcatenate($ENV{'GSDL3SRCHOME'}, 'ext', $gs3_extension, 'perllib'));
101    }
102    }
103    my $path_counter = 0;
104    foreach my $possible_path (@possible_paths) {
105    # we only try adding paths that actually exist
106    if (-d $possible_path) {
107        my $did_add_path = &util::augmentINC($possible_path);
108        if ($did_add_path) {
109        $path_counter++;
110        }
111    }
112    }
113    &_debugPrint('Added ' . $path_counter . ' paths');
114}
115## _addPathsToINC(void) => void #
116
117
118## @function _debugPrint(string, boolean)
119#
120sub _debugPrint
121{
122    my ($message, $newline) = @_;
123    if ($debug) {
124        if (!defined($newline)) {
125            $newline = 1;
126        }
127        print STDERR '[DEBUG] dbutil::' . $message;
128        if ($newline) {
129            print STDERR "\n";
130        }
131    }
132}
133## _debugPrint(string, boolean) => void ##
134
135
136## @function _isDriverLoaded(string) => boolean
137#
138sub _isDriverLoaded
139{
140    my ($dbdriver_name) = @_;
141    (my $dbdriver_file = $dbdriver_name) =~ s/::/\//g;
142    $dbdriver_file .= '.pm';
143    my $result = defined($INC{$dbdriver_file});
144    &_debugPrint('_isDriverLoaded("' . $dbdriver_name . '") => ' . $result);
145    return $result;
146}
147## _isDriverLoaded(string) => boolean ##
148
149## @function _loadDBDriver(string, string)
150#
151sub _loadDBDriver
152{
153    my ($dbdriver_name, $db_filepath) = @_;
154    my $dbdriver;
155    # I've decided (arbitrarily) to use uppercase for driver names since they
156    # are mostly acronyms
157    $dbdriver_name = uc($dbdriver_name);
158    # Ensure the driver has the correct package prefix
159    if ($dbdriver_name !~ /^DBDrivers/) {
160        $dbdriver_name = 'DBDrivers::' . $dbdriver_name;
161    }
162    # We only need to create each driver once
163    if (defined($dbdriver_pool->{$dbdriver_name})) {
164        $dbdriver = $dbdriver_pool->{$dbdriver_name};
165    }
166    else {
167        &_debugPrint('_loadDBDriver() => ' . $dbdriver_name);
168        # Assuming the INC is correctly setup, then this should work nicely
169        # - make sure we have required this dbdriver package
170    eval "require $dbdriver_name";
171    if (&_isDriverLoaded($dbdriver_name)) {
172        $dbdriver_name->import();
173        }
174    # What should we do about drivers that aren't there?
175    else {
176        print STDERR "Error! Failed to load: " . $dbdriver_name . "\n";
177    }
178        # Then initialise and return a new one
179        $dbdriver = $dbdriver_name->new($debug);
180        # Store it for later use
181        $dbdriver_pool->{$dbdriver_name} = $dbdriver;
182    }
183    return $dbdriver;
184}
185## _loadDBDriver(string, string) => BaseDBDriver ##
186
187
188## @function _printTest(string, integer) => void
189#
190sub _printTest
191{
192    my $title = shift(@_);
193    my $result = shift(@_);
194    $test_count++;
195    print " - Test: " . $title . "... ";
196    if ($result) {
197    print "Passed\n";
198    $pass_count++;
199    }
200    else {
201    print "Failed\n";
202    }
203}
204## _printTest(string, integer) => void ##
205
206
207sub _compareHash
208{
209    my $hash1 = shift(@_);
210    my $hash2 = shift(@_);
211    my $str1 = &_hash2str($hash1);
212    my $str2 = &_hash2str($hash2);
213    return ($str1 eq $str2);
214}
215
216sub _hash2str
217{
218    my $hash = shift(@_);
219    my $str = '';
220    foreach my $key (sort keys %{$hash}) {
221    $str .= '{' . $key . '=>{{' . join('},{', @{$hash->{$key}}) . '}}}';
222    }
223    return $str;
224}
225
226
227###############################################################################
228## Public
229###############################################################################
230
231
232## @function main(void) => void
233#
234sub main
235{
236    my $t0 = [gettimeofday()];
237    my $data1 = {'doh' => ['a deer, a female deer'],
238        'ray' => ['a drop of golden sun'],
239        'me'  => ['a name I call myself'],
240        'far' => ['a long, long way to run']};
241    my $data2 = {'sew' => ['a needle pulling thread'],
242         'lah' => ['a note to follow doh'],
243         'tea' => ['a drink with jam and bread'],
244         'doh' => ['which brings us back to']};
245    $test_count = 0;
246    $pass_count = 0;
247    $skip_count = 0;
248    print "===== DBUtils Testing Suite =====\n";
249    print "For each driver specified, run a battery of tests\n";
250    my @drivers;
251    foreach my $arg (@ARGV) {
252    if ($arg =~ /^-+([a-z]+)(=.+)?$/) {
253        my $arg_name = $1;
254        my $arg_value = $2;
255        if ($arg_name eq 'debug') {
256        $debug = 1;
257        }
258    }
259    else {
260        push(@drivers, $arg);
261    }
262    }
263    if (scalar(@drivers)) {
264    # Ensure the Perl can load the drivers from all the typical places
265    &_addPathsToINC();
266    foreach my $driver_name (@drivers) {
267        my $t1 = [gettimeofday()];
268        print "=== Testing: " . $driver_name . " ===\n";
269        my $driver = _loadDBDriver($driver_name);
270        my $db_path = $driver->get_infodb_file_path('test-doc','/tmp/');
271        print " - Path: " . $db_path . "\n";
272        # 1. Open handle
273        my $db_handle = $driver->open_infodb_write_handle($db_path);
274        &_printTest('opening handle', (defined $db_handle));
275        # 2a. Write entry
276        $driver->write_infodb_entry($db_handle, 'Alpha', $data1);
277        &_printTest('writing entry', 1);
278        # 2b. Write raw entry
279        my $raw_data = $driver->convert_infodb_hash_to_string($data1);
280        $driver->write_infodb_rawentry($db_handle, 'Beta', $raw_data);
281        &_printTest('writing raw entry', 1);
282        # 3. Close handle
283        $driver->close_infodb_write_handle($db_handle);
284        if ($driver->supportsPersistentConnection()) {
285        $test_count += 1;
286        $skip_count += 1;
287        print " - Skipping test as persistent drivers delay 'close'.\n";
288        }
289        else {
290        &_printTest('closing handle', (tell($db_handle) < 1));
291        }
292        if (!$driver->writeOnly()) {
293        # 4a. Read entry
294        my $data3 = $driver->read_infodb_entry($db_path, 'Alpha');
295        &_printTest('read entry', &_compareHash($data1, $data3));
296        # 4b. Read raw entry
297        my $raw_data4 = $driver->read_infodb_rawentry($db_path, 'Beta');
298        my $data4 = $driver->convert_infodb_string_to_hash($raw_data4);
299        &_printTest('read raw entry', &_compareHash($data1, $data4));
300        # 5. Read keys
301        my $keys1 = {};
302        $driver->read_infodb_keys($db_path, $keys1);
303        &_printTest('read keys', (defined $keys1->{'Alpha'} && defined $keys1->{'Beta'}));
304        # 6. Set entry
305        if ($driver->supportsSet()) {
306            my $status = $driver->set_infodb_entry($db_path, 'Alpha', $data2);
307            &_printTest('set entry (1)', ($status >= 0));
308            my $data5 = $driver->read_infodb_entry($db_path, 'Alpha');
309            &_printTest('set entry (2)', &_compareHash($data2, $data5));
310        }
311        else {
312            $test_count += 2;
313            $skip_count += 2;
314            print " - Skipping 2 tests as 'set' is not supported by this driver.\n";
315        }
316        # 7. Delete entry
317        my $db_handle2 = $driver->open_infodb_write_handle($db_path, 'append');
318        $driver->delete_infodb_entry($db_handle2, 'Alpha');
319        $driver->close_infodb_write_handle($db_handle2);
320        my $keys2 = {};
321        $driver->read_infodb_keys($db_path, $keys2);
322        &_printTest('delete entry', ((!defined $keys2->{'Alpha'}) && (defined $keys2->{'Beta'})));
323        }
324        else
325        {
326        $test_count += 6;
327        $skip_count += 6;
328        print " - Skipping 6 tests as driver is write-only.\n";
329        }
330        # 8. Remove test db
331        unlink($db_path);
332        my $t2 = [gettimeofday()];
333        my $elapsed1 = tv_interval($t1, $t2);
334        print " - Testing took " . $elapsed1 . " seconds\n";
335    }
336    print "===== Results =====\n";
337    print "Drivers Tested: " . scalar(@drivers) . "\n";
338    print "Tests Run:      " . $test_count . "\n";
339    print "Tests Passed:   " . $pass_count . "\n";
340    print "Tests Failed:   " . ($test_count - $pass_count - $skip_count) . "\n";
341    print "Tests Skipped:  " . $skip_count . "\n";
342    }
343    else
344    {
345    print "Warning! No drivers specified - expected as arguments to call\n";
346    }
347    my $t3 = [gettimeofday()];
348    my $elapsed2 = tv_interval($t0, $t3);
349    print "===== Complete in " . $elapsed2 . " seconds =====\n";
350    print "\n";
351    exit(0);
352}
353## main(void) => void
354
355
356## @function close_infodb_write_handle(string, *) => void
357#
358sub close_infodb_write_handle
359{
360  my $infodb_type = shift(@_);
361  my $driver = _loadDBDriver($infodb_type);
362  $driver->close_infodb_write_handle(@_);
363}
364## close_infodb_write_handle(string, *) => void ##
365
366
367## @function delete_infodb_entry(string, *) => void
368#
369sub delete_infodb_entry
370{
371    my $infodb_type = shift(@_);
372    my $driver = _loadDBDriver($infodb_type);
373    $driver->delete_infodb_entry(@_);
374}
375## delete_infodb_entry(string, *) => void ##
376
377
378## @function mergeDatabases(string, *) => integer
379#
380sub mergeDatabases
381{
382    my $infodb_type = shift(@_);
383    my $driver = _loadDBDriver($infodb_type);
384    my $status = $driver->mergeDatabases(@_);
385    return $status;
386}
387## mergeDatabases(string, *) => integer ##
388
389
390## @function get_default_infodb_type(void) => string
391#
392sub get_default_infodb_type
393{
394  # The default is GDBM so everything works the same for existing collections
395  # To use something else, specify the "infodbtype" in the collection's collect.cfg file
396  return 'gdbm';
397}
398## get_default_infodb_type(void) => string ##
399
400
401## @function get_infodb_file_path(string, *) => string
402#
403sub get_infodb_file_path
404{
405    my $infodb_type = shift(@_);
406    my $driver = _loadDBDriver($infodb_type);
407    my $infodb_file_path = $driver->get_infodb_file_path(@_);
408    return $infodb_file_path;
409}
410## get_infodb_file_path(string, *) => string ##
411
412
413## @function open_infodb_write_handle(string, *) => filehandle
414#
415sub open_infodb_write_handle
416{
417    my $infodb_type = shift(@_);
418    my $driver = _loadDBDriver($infodb_type);
419    my $infodb_handle = $driver->open_infodb_write_handle(@_);
420    return $infodb_handle;
421}
422## open_infodb_write_handle(string, *) => filehandle ##
423
424
425## @function read_infodb_file(string, *) => void
426#
427sub read_infodb_file
428{
429    my $infodb_type = shift(@_);
430    my $driver = _loadDBDriver($infodb_type);
431    $driver->read_infodb_file(@_);
432}
433## read_infodb_file(string, *) => void ##
434
435
436## @function read_infodb_keys(string, *) => void
437#
438sub read_infodb_keys
439{
440    my $infodb_type = shift(@_);
441    my $driver = _loadDBDriver($infodb_type);
442    $driver->read_infodb_keys(@_);
443}
444## read_infodb_keys(string, *) => void ##
445
446
447## @function read_infodb_entry(string, *) => hashmap
448#
449sub read_infodb_entry
450{
451    my $infodb_type = shift(@_);
452    my $driver = _loadDBDriver($infodb_type);
453    my $infodb_entry = $driver->read_infodb_entry(@_);
454    return $infodb_entry;
455}
456## read_infodb_entry(string, *) => hashmap ##
457
458
459## @function read_infodb_rawentry(string, *) => string
460#
461sub read_infodb_rawentry
462{
463    my $infodb_type = shift(@_);
464    my $driver = _loadDBDriver($infodb_type);
465    my $raw_infodb_entry = $driver->read_infodb_rawentry(@_);
466    return $raw_infodb_entry;
467}
468## read_infodb_rawentry(string, *) => string ##
469
470
471## @function set_infodb_entry(string, *) => integer
472#
473sub set_infodb_entry
474{
475    my $infodb_type = shift(@_);
476    my $driver = _loadDBDriver($infodb_type);
477    my $status = $driver->set_infodb_entry(@_);
478    return $status;
479}
480## set_infodb_entry(string, *) => integer ##
481
482
483## @function supportDatestamp(string) => boolean
484#
485sub supportsDatestamp
486{
487    my $infodb_type = shift(@_);
488    my $driver = _loadDBDriver($infodb_type);
489    my $supports_datestamp = $driver->supportsDatestamp();
490    return $supports_datestamp;
491}
492## supportsDatestamp(string) => boolean ##
493
494
495## @function supportMerge(string) => boolean
496#
497sub supportsMerge
498{
499    my $infodb_type = shift(@_);
500    my $driver = _loadDBDriver($infodb_type);
501    my $supports_merge = $driver->supportsMerge();
502    return $supports_merge;
503}
504## supportsMerge(string) => boolean ##
505
506
507## @function supportRSS(string) => boolean
508#
509sub supportsRSS
510{
511    my $infodb_type = shift(@_);
512    my $driver = _loadDBDriver($infodb_type);
513    my $supports_rss = $driver->supportsRSS();
514    return $supports_rss;
515}
516## supportsRSS(string) => boolean ##
517
518
519## @function write_infodb_entry(string, *) => void
520#
521sub write_infodb_entry
522{
523    my $infodb_type = shift(@_);
524    my $driver = _loadDBDriver($infodb_type);
525    $driver->write_infodb_entry(@_);
526}
527## write_infodb_entry(string, *) => void ##
528
529
530## @function write_infodb_rawentry(string, *) => void
531#
532sub write_infodb_rawentry
533{
534    my $infodb_type = shift(@_);
535    my $driver = _loadDBDriver($infodb_type);
536    $driver->write_infodb_rawentry(@_);
537}
538## write_infodb_rawentry(string, *) => void ##
539
5401;
Note: See TracBrowser for help on using the browser.