root/main/trunk/greenstone2/perllib/gssql.pm @ 32541

Revision 32541, 19.0 KB (checked in by ak19, 9 months ago)

Using proper parameters to GreenstoneSQLPlugin/Plugout instead of hardcoded values for params.

Line 
1###########################################################################
2#
3# gssql.pm -- DBI for SQL related utility functions used by
4# GreenstoneSQLPlugout and hereafter by GreenstoneSQLPlugin too.
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright (C) 1999 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26
27package gssql;
28
29use strict;
30no strict 'refs';
31no strict 'subs';
32
33use DBI; # the central package for this module used by GreenstoneSQL Plugout and Plugin
34
35# Need params_map keys:
36# - collection_name
37# - db_encoding (db content encoding) - MySQL can set this at server, db, table levels. For MySQL
38# we set the enc during connect at server level. Not sure whether other DB's support it at the
39# same levels.
40
41# For connection to MySQL, need:
42#  - db_driver, db_client_user, db_client_pwd, db_host, (db_port not used at present)
43# So these will be parameterised, but in a hashmap, for just the connect method.
44
45# Parameterise (one or more methods may use them):
46# - build_mode (like removeold)
47# - db_name (which is the GS3 sitename)
48
49# TODO: add infrastructure for db_port, AutoCommit etc
50# For port, see https://stackoverflow.com/questions/2248665/perl-script-to-connect-to-mysql-server-port-3307
51
52sub new
53{
54 
55    my $class = shift(@_);
56   
57    my ($params_map) = @_;
58   
59    # library_url: to be specified on the cmdline if not using a GS-included web server
60    # the GSDL_LIBRARY_URL env var is useful when running cmdline buildcol.pl in the linux package manager versions of GS3
61   
62    # https://stackoverflow.com/questions/7083453/copying-a-hashref-in-perl
63    # Making a shallow copy works, and can handle unknown params:
64    #my $self = $params_map;
65
66    # but being explicit for class params needed for MySQL:
67    my $self = {
68    'collection_name' => $params_map->{'collection_name'},
69    'db_encoding' => $params_map->{'db_encoding'}
70    };
71
72    # (My)SQL doesn't like tables with - (hyphens) in their names
73    my $coll_name = $params_map->{'collection_name'};
74    $coll_name =~ s/-/_/g;
75    $self->{'tablename_prefix'} = $coll_name;
76
77    return bless($self, $class);
78}
79
80
81#################################
82
83# Database access related functions
84# http://g2pc1.bu.edu/~qzpeng/manual/MySQL%20Commands.htm
85# https://www.guru99.com/insert-into.html
86
87# TODO Q: What on cancelling a build: delete table? But what if it was a rebuild and the rebuild is cancelled (not the original build)?
88# Do we create a copy of the orig database as backup, then start populating current db, and if cancelled, delete current db and RENAME backup table to current?
89# https://stackoverflow.com/questions/3280006/duplicating-a-mysql-table-indexes-and-data
90# BUT what if the table is HUGE? (Think of a collection with millions of docs.) Huge overhead in copying?
91# The alternative is we just quit on cancel, but then: cancel could leave the table in a partial committed state, with no way of rolling back.
92# Unless they do a full rebuild, which will recreate the table from scratch?
93# SOLUTION-> rollback transaction on error, see https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
94# But then should set AutoCommit to off on connection, and remember to commit every time
95
96#################
97# Database functions that use the perl DBI module (with the DBD driver module for mysql)
98#################
99
100################### BASIC DB OPERATIONS ##################
101
102# THE NEW DB FUNCTIONS
103# NOTE: FULLTEXT is a reserved keyword in (My)SQL. So we can't name a table or any of its columns "fulltext".
104# https://dev.mysql.com/doc/refman/5.5/en/keywords.html
105
106# TODO: Consider AutoCommit status (and Autocommit off allowing commit or rollback for GS coll build cancel) later
107
108# TODO: where should the defaults for these params be, here or in GS-SQLPlugin/Plugout?
109sub connect_to_db {
110    my $self= shift (@_);
111    my ($params_map) = @_;
112    my $db_enc = $self->{'db_encoding'} || "utf8";
113
114    # these are the params for connecting to MySQL
115    my $db_driver = $params_map->{'db_driver'} || "mysql";
116    my $db_user = $params_map->{'db_client_user'} || "root";
117    my $db_pwd = $params_map->{'db_client_pwd'}; # even if undef, we'll see a sensible error message
118                                           # when connect fails
119    my $db_host = $params_map->{'db_host'} || "127.0.0.1";
120    # localhost doesn't work for us, but 127.0.0.1 works
121    # https://metacpan.org/pod/DBD::mysql
122    # "The hostname, if not specified or specified as '' or 'localhost', will default to a MySQL server
123    # running on the local machine using the default for the UNIX socket. To connect to a MySQL server
124    # on the local machine via TCP, you must specify the loopback IP address (127.0.0.1) as the host."
125    #my $connect_str = "dbi:$db_driver:database=$db_name;host=$db_host";
126    my $connect_str = "dbi:$db_driver:host=$db_host"; # don't provide db - allows checking the db exists
127    my $dbh = DBI->connect("$connect_str", $db_user, $db_pwd,
128               {
129                   ShowErrorStatement => 1, # more informative as DBI will append failed SQL stmt to error message
130                   PrintError => 1, # on by default, but being explicit
131                   RaiseError => 0, # off by default, but being explicit
132                   AutoCommit => 1 # on by default, but being explicit
133               });
134
135    if(!$dbh) {
136    # NOTE, despite handle dbh being undefined, error code will be in DBI->err
137    return 0;   
138    }
139
140    # set encoding https://metacpan.org/pod/DBD::mysql
141    # https://dev.mysql.com/doc/refman/5.7/en/charset.html
142    # https://dev.mysql.com/doc/refman/5.7/en/charset-conversion.html
143    # Setting the encoding at db server level.
144    # Not sure if this command is mysql specific:
145    my $stmt = "set NAMES '" . $db_enc . "'";
146    $dbh->do($stmt) || warn("Unable to set charset encoding at db server level to: " . $db_enc . "\n");
147   
148    # if we're here, then connection succeeded, store handle
149    $self->{'db_handle'} = $dbh;
150    return 1;
151}
152
153# will attempt to load the specified db and the <coll>_metadata and <coll>_fulltxt for this
154# collection, or create any of these (db, tables) that don't yet exist. At the end
155# it will have loaded the requested database (in MySQL: "use <db>;")
156sub load_db_and_tables {
157    my $self= shift (@_);
158    my ($db_name, $build_mode) = @_;
159    my $dbh = $self->{'db_handle'};
160   
161    # perl DBI switch database: https://www.perlmonks.org/?node_id=995434
162    # do() returns undef on error.
163    # connection succeeded, try to load our database. If that didn't work, attempt to create db
164    my $success = $dbh->do("use $db_name");
165   
166    if(!$success && $dbh->err == 1049) { # "Unknown database" error has code 1049 (mysql only?) meaning db doesn't exist yet
167    # attempt to create the db and its tables
168    $self->create_db($db_name) || return 0;
169
170    print STDERR "@@@ CREATED DATABASE $db_name\n";
171   
172    # once more attempt to use db, now that it exists
173    $dbh->do("use $db_name") || return 0;
174    #$dbh->do("use localsite") or die "Error (code" . $dbh->err ."): " . $dbh->errstr . "\n";
175
176    # attempt to create tables in current db
177    $self->create_metadata_table() || return 0;
178    $self->create_fulltext_table() || return 0;
179
180    $success = 1;
181    }
182    elsif($success) { # database existed and loaded successfully, but
183    # before proceeding check that the current collection's tables exist
184
185    print STDERR "@@@ DATABASE $db_name EXISTED\n";
186   
187
188    if($build_mode eq "removeold") {
189        $self->delete_collection_tables();
190    }
191
192    # use existing tables if any
193    # attempt to create tables in current db   
194    if($build_mode eq "removeold" || !$self->table_exists($self->get_metadata_table_name())) {
195        $self->create_metadata_table() || return 0;
196    } else {
197        print STDERR "@@@ Meta table exists\n";
198    }
199    if($build_mode eq "removeold" || !$self->table_exists($self->get_fulltext_table_name())) {
200        $self->create_fulltext_table() || return 0;
201    } else {
202        print STDERR "@@@ Fulltxt table exists\n";
203    }
204   
205    }
206   
207    return $success; # could still return 0, if database failed to load with an error code != 1049
208}
209
210# GreenstoneSQLPlugin calls this method to load an existing db.
211# This will terminate if the db does not exist. Unlike load_db_and_tables() above, used by
212# GreenstoneSQLPlugout, this method will not attempt to create the requested db (nor its tables)
213# TODO: GS SQLPlugin is called before GS SQLPlugout and attempts to use_db() - called in plugin's
214# init() method. This will fail if the db does not exist. Ideally want our plugin only called
215# during buildcol.pl
216sub use_db {
217    my $self= shift (@_);
218    my ($db_name) = @_;
219    my $dbh = $self->{'db_handle'};
220   
221    # perl DBI switch database: https://www.perlmonks.org/?node_id=995434
222    # do() returns undef on error.
223    # connection succeeded, try to load our database. If that didn't work, attempt to create db
224    return $dbh->do("use $db_name") || warn();
225}
226
227# disconnect from db - https://metacpan.org/pod/DBI#disconnect
228# TODO: make sure to have committed or rolled back before disconnect
229# and that you've call finish() on statement handles if any fetch remnants remain
230sub disconnect_from_db {
231    my $self= shift (@_);   
232    my $dbh = $self->{'db_handle'};
233
234    # make sure any active stmt handles are finished
235    # NO: "When all the data has been fetched from a SELECT statement, the driver will automatically call finish for you. So you should not call it explicitly except when you know that you've not fetched all the data from a statement handle and the handle won't be destroyed soon."
236   
237    #$meta_sth = $self->{'metadata_prepared_insert_statement_handle'};
238    #$txt_sth = $self->{'fulltxt_prepared_insert_statement_handle'};
239    #$meta_sth->finish() if($meta_sth);
240    #$txt_sth->finish() if($txt_sth);
241   
242    my $rc = $dbh->disconnect or warn $dbh->errstr; # The handle is of little use after disconnecting. Possibly PrintError already prints a warning and this duplicates it?
243    return $rc;
244}
245
246sub create_db {
247    my $self= shift (@_);
248    my $db_name = $self->{'db_name'};
249    my $dbh = $self->{'db_handle'};
250   
251    # https://stackoverflow.com/questions/5025768/how-can-i-create-a-mysql-database-from-a-perl-script
252    return $dbh->do("create database $db_name"); # do() will return undef on fail, https://metacpan.org/pod/DBI#do
253}
254
255
256sub create_metadata_table {
257    my $self= shift (@_);
258    my $dbh = $self->{'db_handle'};
259   
260    my $table_name = $self->get_metadata_table_name();
261
262    # If using an auto incremented primary key:
263    my $stmt = "CREATE TABLE $table_name (id INT NOT NULL AUTO_INCREMENT, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, metaname VARCHAR(127) NOT NULL, metavalue VARCHAR(1023) NOT NULL, PRIMARY KEY(id));";
264    return $dbh->do($stmt);
265}
266
267# TODO: Investigate: https://dev.mysql.com/doc/search/?d=10&p=1&q=FULLTEXT
268# 12.9.1 Natural Language Full-Text Searches
269# to see whether we have to index the 'fulltxt' column of the 'fulltext' tables
270# or let user edit this file, or add it as another option
271sub create_fulltext_table {
272    my $self= shift (@_);
273    my $dbh = $self->{'db_handle'};
274   
275    my $table_name = $self->get_fulltext_table_name();
276
277    # If using an auto incremented primary key:
278    my $stmt = "CREATE TABLE $table_name (id INT NOT NULL AUTO_INCREMENT, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, fulltxt LONGTEXT, PRIMARY KEY(id));";
279    return $dbh->do($stmt);
280
281}
282
283# "IF EXISTS is used to prevent an error from occurring if the database does not exist. ... DROP DATABASE returns the number of tables that were removed. The DROP DATABASE statement removes from the given database directory those files and directories that MySQL itself may create during normal operation.Jun 20, 2012"
284#MySQL 8.0 Reference Manual :: 13.1.22 DROP DATABASE Syntax
285# https://dev.mysql.com/doc/en/drop-database.html
286sub delete_collection_tables {
287    my $self= shift (@_);
288    my $dbh = $self->{'db_handle'};
289   
290    print STDERR "### Build mode is removeold, so deleting tables for current collection\n";
291   
292    # drop table <tablename>
293    my $table = $self->get_metadata_table_name();
294    $dbh->do("drop table $table") || warn("@@@ Couldn't delete $table");
295    $table = $self->get_fulltext_table_name();
296    $dbh->do("drop table $table") || warn("@@@ Couldn't delete $table");
297}
298
299# Don't call this: it will delete the meta and full text tables for ALL collections in $db_name (localsite by default)!
300# This method is just here for debugging (for testing creating a database when there is none)
301sub _delete_database {
302    my $self= shift (@_);
303    my ($db_name) = @_;
304    my $dbh = $self->{'db_handle'};
305   
306    # "drop database dbname"
307    $dbh->do("drop database $db_name") || return 0;
308
309    return 1;
310}
311
312
313########################### DB STATEMENTS ###########################
314
315# USEFUL: https://metacpan.org/pod/DBI
316# "Many methods have an optional \%attr parameter which can be used to pass information to the driver implementing the method. Except where specifically documented, the \%attr parameter can only be used to pass driver specific hints. In general, you can ignore \%attr parameters or pass it as undef."
317
318
319# https://www.guru99.com/insert-into.html
320# and https://dev.mysql.com/doc/refman/8.0/en/example-auto-increment.html
321#     for inserting multiple rows at once
322# https://www.perlmonks.org/bare/?node_id=316183
323# https://metacpan.org/pod/DBI#do
324# https://www.quora.com/What-is-the-difference-between-prepare-and-do-statements-in-Perl-while-we-make-a-connection-to-the-database-for-executing-the-query
325# https://docstore.mik.ua/orelly/linux/dbi/ch05_05.htm
326
327# https://metacpan.org/pod/DBI#performance
328# 'The q{...} style quoting used in this example avoids clashing with quotes that may be used in the SQL statement. Use the double-quote like qq{...} operator if you want to interpolate variables into the string. See "Quote and Quote-like Operators" in perlop for more details.'
329sub prepare_insert_metadata_row_stmthandle {
330    my $self = shift (@_);   
331    #my ($did, $sid, $metaname, $metavalue) = @_;
332    my $dbh = $self->{'db_handle'};
333   
334    my $tablename = $self->get_metadata_table_name();
335
336    #my $stmt = "INSERT INTO $tablename (did, sid, metaname, metavalue) VALUES ('$did', '$sid', '$metaname', '$metavalue');"; # ?, ?, ?, ?
337
338    # using qq{} since we want $tablename placeholder to be filled in
339    # returns Statement Handle object!
340    my $sth = $dbh->prepare(qq{INSERT INTO $tablename (did, sid, metaname, metavalue) VALUES (?, ?, ?, ?)}) || warn("Could not prepare insert statement for metadata table\n");
341
342    print STDERR "@@@@ Prepared meta insert statement: ".$sth->{'Statement'}."\n";
343   
344    return $sth;
345}
346
347sub prepare_insert_fulltxt_row_stmthandle {
348    my $self = shift (@_);
349    #my ($did, $sid, $fulltext) = @_;
350    my $dbh = $self->{'db_handle'};
351   
352    my $tablename = $self->get_fulltext_table_name();
353
354    #my $stmt = "INSERT INTO $tablename (did, sid, fulltxt) VALUES ('$did', '$sid', '$fulltext');"; ?, ?, ?
355
356    # using qq{} since we want $tablename placeholder to be filled in
357    # returns Statement Handle object!
358    my $sth = $dbh->prepare(qq{INSERT INTO $tablename (did, sid, fulltxt) VALUES (?, ?, ?)}) || warn("Could not prepare insert statement for fulltxt table\n");
359   
360    print STDERR "@@@@ Prepared fulltext insert statement: ".$sth->{'Statement'}."\n";
361   
362    return $sth;
363}
364
365
366## The 2 select statements used by GreenstoneSQLPlugin
367
368# Returns the statement handle that prepared and executed
369# a "SELECT * FROM <COLL>_metadata WHERE did = $oid" SQL statement.
370# Caller can call fetchrow_array() on returned statement handle, $sth
371# Have to use prepare() and execute() instead of do() since do() does
372# not allow for fetching result set thereafter:
373# do(): "This method  is typically most useful for non-SELECT statements that either cannot be prepared in advance (due to a limitation of the driver) or do not need to be executed repeatedly. It should not be used for SELECT statements because it does not return a statement handle (so you can't fetch any data)." https://metacpan.org/pod/release/TIMB/DBI-1.634_50/DBI.pm#do
374sub select_from_metatable_matching_docid {
375    my $self= shift (@_);
376    my ($oid) = @_;
377   
378    my $dbh = $self->{'db_handle'};
379    my $meta_table = $self->get_metadata_table_name();
380   
381    my $sth = $dbh->prepare(qq{SELECT * FROM $meta_table WHERE did = ?});
382    $sth->execute( $oid ); # will print msg on fail
383   
384    return $sth; # caller can call fetchrow_array() on returned statement handle, sth
385}
386
387# Returns the statement handle that prepared and executed
388# a "SELECT * FROM <COLL>_metadata WHERE did = $oid" SQL statement.
389# Caller can call fetchrow_array() on returned statement handle, $sth
390sub select_from_texttable_matching_docid {
391    my $self= shift (@_);
392    my ($oid) = @_;
393   
394    my $dbh = $self->{'db_handle'};
395    my $fulltxt_table = $self->get_fulltext_table_name();
396   
397    my $sth = $dbh->prepare(qq{SELECT * FROM $fulltxt_table WHERE did = ?});
398    $sth->execute( $oid ); # will print msg on fail
399   
400    return $sth; # caller can call fetchrow_array() on returned statement handle, sth
401}
402
403
404# Can call this after connection succeeded to get the database handle, dbh,
405# if any specific DB operation (SQL statement, create/delete)
406# needs to be executed that is not already provided as a method of this class.
407sub get_db_handle {
408    my $self= shift (@_);
409    return $self->{'db_handle'};
410}
411
412################ HELPER METHODS ##############
413
414# More basic helper methods
415sub get_metadata_table_name {
416    my $self= shift (@_);
417    my $table_name = $self->{'tablename_prefix'} . "_metadata";
418    return $table_name;
419}
420
421# FULLTEXT is a reserved keyword in (My)SQL. https://dev.mysql.com/doc/refman/5.5/en/keywords.html
422# So we can't name a table or any of its columns "fulltext". We use "fulltxt" instead.
423sub get_fulltext_table_name {
424    my $self= shift (@_);
425    my $table_name = $self->{'tablename_prefix'} . "_fulltxt";
426    return $table_name;
427}
428
429
430# I can get my version of table_exists to work, but it's not so ideal
431# Interesting that MySQL has non-standard command to CREATE TABLE IF NOT EXISTS and DROP TABLE IF EXISTS,
432# see https://www.perlmonks.org/bare/?node=DBI%20Recipes
433#    The page further has a table_exists function that could work with proper comparison
434# TODO: Couldn't get the first solution at https://www.perlmonks.org/bare/?node_id=500050 to work though
435sub table_exists {
436    my $self = shift (@_);
437    my $dbh = $self->{'db_handle'};
438    my ($table_name) = @_;
439
440    my @table_list = $dbh->tables;
441    #my $tables_str = @table_list[0];
442    foreach my $table (@table_list) {
443    return 1 if ($table =~ m/$table_name/);
444    }
445    return 0;
446}
447
4481;
Note: See TracBrowser for help on using the browser.