Index: /trunk/gsdl/etc/packages/example.dbi
===================================================================
--- /trunk/gsdl/etc/packages/example.dbi	(revision 4429)
+++ /trunk/gsdl/etc/packages/example.dbi	(revision 4429)
@@ -0,0 +1,77 @@
+# -*- perl -*-
+# This is an example configuration file for greenstone's DBPlug
+# (DataBase plugin).
+#
+# This config file should be a fragment of perl, specifying how to
+# get records out of a database. This uses perl's DBI module, which
+# includes back-ends for mysql, postgresql, comma separated values (CSV),
+# MS Excel, ODBC, sybase, etc...
+#
+# You need to install the separate backends - For example, on debian you
+# need the libdbi-perl base package and then the libdbd-mysql-perl,
+# libdbd-csv-perl, libdbd-pg-perl and/or libdbd-sybase-perl (etc) packages.
+
+
+# debugging. Uncomment this line if you want extra information printed
+# out during greenstone's import process.
+#
+$dbplug_debug=1;
+
+
+# If these are commented out, then we will use the plugin's defaults.
+# However, you might want to override them here if you know what they
+# are for this particular database
+# run "$ pluginfo.pl DBPlug | less" to see a list of valid encodings
+# Language metadata isn't really necessary unless you want indexes by language
+# However, Encoding is important, for converting data to utf8
+#
+$language = "en";
+$encoding = "iso_8859_1";
+
+
+# Data Source.
+# This is going to be dependent on which back-end you use.
+# Eg, for postgres, it is "DBI:Pg:dbname=<database>".
+# Case is important!
+#
+#$db='';
+
+# If you need a username and password to connect to this database,
+# uncomment these lines and fill in the details
+#
+#$username='';
+#$password='';
+
+# sql query required to get your rows out of the database. Each row will
+# become a separate greenstone document.
+#
+#$sql_query = 'SELECT id, name,  data FROM table1 JOIN table2 WHERE something';
+
+
+# a map of database field names to Greenstone metadata field names.
+# You should have a "Title" field and a "text" field, at a minimum.
+# If you want to give records a unique identifier from a database field
+# rather than have greenstone perform a hash on the document, also provide
+# an "Identifier" metadata field.
+# If you do map something to Identifier, make sure that the field is unique
+# for each record, and also note that because greenstone does special things
+# if this field is only numeric, DBPlug will add some letters to the front
+# if this is the case.
+#
+#%db_to_greenstone_fields=(
+#			  "name" => "Title",
+#			  "data" => "text",
+#			  "id" => "Identifier"
+#			  );
+
+
+# finally, you can declare a subroutine here to process the "text"
+# in case you want to do any mark up or processing of it. 
+# For example, you might want to do HTML formatting.
+#
+#sub text_callback {
+#    my $text=shift;
+#    # do markup stuff...
+#    return $text;
+#}
+		 
Index: /trunk/gsdl/perllib/plugins/DBPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/DBPlug.pm	(revision 4429)
+++ /trunk/gsdl/perllib/plugins/DBPlug.pm	(revision 4429)
@@ -0,0 +1,298 @@
+###########################################################################
+#
+# DBPlug.pm -- plugin to import records from a database
+# 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2003 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+#
+# See <GSDLHOME>/etc/packages/example.dbi for an example config file!!
+#
+
+# Written by John McPherson for the NZDL project
+# Mar, Apr 2003
+
+package DBPlug;
+
+use BasPlug;
+use unicode;
+use parsargv;
+
+use DBI; # database independent stuff
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+my $arguments = [ { 'name' => "process_exp",
+						  'desc' => "A perl regular expression to match against filenames. Matching filenames will be processed by this plugin. Each plugin has its own default process_exp. e.g HTMLPlug defaults to '(?i)\.html?\$' i.e. all documents ending in .htm or .html (case-insensitive).",
+						  'type' => "string",
+						  'deft' => q^(?i)\.dbi$^,
+						  'reqd' => "no" } ,
+						{ 'name' => "title_sub",
+						  'desc' => "Substitution expression to modify string stored as Title. Used by, for example, PSPlug to remove \"Page 1\" etc from text used as the title.",
+						  'type' => "string",
+						  'reqd' => "no" }];
+
+my $options = { 'name'     => "DBPlug",
+	             'desc'     => "Uses records from a database as documents.",
+					 'inherits' => "yes",
+					 'args'     => $arguments };
+
+sub print_usage {
+    print STDERR "\n  usage: plugin DBPlug [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "\n";
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new BasPlug ($class, @_);
+
+    my $option_list = $self->{'option_list'};
+    push( @{$option_list}, $options );
+
+    if (!parsargv::parse(\@_, 
+			 q^title_sub/.*/^, \$self->{'title_sub'},
+			 "allow_extra_options")) {
+	print STDERR "\nIncorrect options passed to DBPlug, check your collect.cfg configuration file\n";
+	&print_usage();
+        die "\n";
+    }
+
+
+    return bless $self, $class;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.dbi$^;
+}
+# we don't have a per-greenstone document process() function!
+sub process {
+
+}
+
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    # see if we can handle the passed file...
+    my $filename = $file;
+    $filename = &util::filename_cat ($base_dir, $file) if $base_dir =~ /\w/;
+    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
+	# this plugin can't process this file type...
+        return undef;
+    }
+
+    my $outhandle = $self->{'outhandle'};
+    my $verbosity = $self->{'verbosity'};
+
+    print $outhandle "DBPlug: processing $file\n" 
+	if $self->{'verbosity'} > 1;
+    
+    # calculate the document hash, for document ids
+    my $hash="";
+
+    my $osexe = &util::get_os_exe();
+    my $hashfile_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",
+					   $ENV{'GSDLOS'},"hashfile$osexe");
+    if (-e "$hashfile_exe") {
+	$hash = `hashfile$osexe \"$filename\"`;
+	$hash =~ /:\s*([0-9a-f]+)/i;
+	$hash="HASH$1";
+    }
+
+
+    # default options - may be overridden by config file
+    my $language=undef;
+    my $encoding=undef;
+    my $dbplug_debug=0;
+    my $username='';
+    my $password='';
+
+    # these settings must be set by the config file:
+    my $db=undef;
+
+# get id of pages from "nonempty", get latest version number from "recent", and
+# then get pagename from "page" and content from "version" !
+
+    my $sql_query = undef ;
+
+    my %db_to_greenstone_fields=();
+
+    # read in config file.
+    eval `cat $filename`;
+
+    if (!defined($db)) {
+	print $outhandle "DBPlug: error: $filename does not specify a db!\n";
+	return 0;
+    }
+    if (!defined($sql_query)) {
+    	print $outhandle "DBPlug: error: no SQL query specified!\n";
+	return 0;
+    }
+    # connect to database
+    my $dbhandle=DBI->connect($db, $username, $password);
+
+    if (!defined($dbhandle)) {
+	die "DBPlug: could not connect to database, exiting.\n";
+    }
+    if (defined($dbplug_debug) && $dbplug_debug==1) {
+	print $outhandle "DBPlug (debug): connected ok\n";
+    }
+
+    my $statement_hand=$dbhandle->prepare($sql_query);
+    $statement_hand->execute;
+
+    # get the array-ref for the field names and cast it to array
+    my @field_names;
+    @field_names=@{ $statement_hand->{NAME} };
+
+    foreach my $fieldname (@field_names) {
+	if (defined($db_to_greenstone_fields{$fieldname})) {
+	    if (defined($dbplug_debug) && $dbplug_debug==1) {
+		print $outhandle "DBPlug (debug): mapping db field "
+		    . "'$fieldname' to "
+			. $db_to_greenstone_fields{$fieldname} . "\n";
+	    }
+	    $fieldname=$db_to_greenstone_fields{$fieldname};
+	}
+    }
+
+
+#    print "DBPlug: names: " . join (", ", @field_names) . ".\n";
+    # get rows
+
+    my $count = 0;
+    my @row_array;
+
+    @row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
+    
+    while (scalar(@row_array)) {
+	if (defined($dbplug_debug) && $dbplug_debug==1) {
+	    print $outhandle "DBPlug (debug): retrieved a row from query\n";
+	}
+
+	# create a new document
+	my $doc_obj = new doc ($filename, "indexed_doc");
+	$doc_obj->set_OIDtype ($processor->{'OIDtype'});
+	my $cursection = $doc_obj->get_top_section();
+
+	if (defined($language)) {
+	    # if not set in config file, will use BasPlug's default
+	    $doc_obj->add_utf8_metadata($cursection, "Language", $language);
+	}
+	if (defined($encoding)) {
+	    # if not set in config file, will use BasPlug's default
+	    $doc_obj->add_utf8_metadata($cursection, "Encoding", $encoding);
+	}
+	$doc_obj->add_utf8_metadata($cursection,
+				    "Source", &ghtml::dmsafe($db));
+	if ($self->{'cover_image'}) {
+	    $self->associate_cover_image($doc_obj, $filename);
+	}
+	
+
+	# include any metadata passed in from previous plugins 
+	# note that this metadata is associated with the top level section
+	$self->extra_metadata ($doc_obj, $cursection,
+			       $metadata);
+
+	# do any automatic metadata extraction
+	$self->auto_extract_metadata ($doc_obj);
+
+	my $unique_id=undef;
+
+	foreach my $fieldname (@field_names) {
+	    my $fielddata=shift @row_array;
+	    # use the specified encoding, defaulting to utf-8
+	    if (defined($encoding) && $encoding ne "ascii"
+	    	&& $encoding ne "utf8") {
+	      $fielddata=&unicode::unicode2utf8(
+	      	&unicode::convert2unicode($encoding, \$fielddata)
+					       );
+	    }
+	    if ($fieldname eq "text") {
+		# see if we have a text_callback() function defined
+		if (defined(&text_callback)) {
+		    $fielddata=text_callback($fielddata);
+		}
+		# add as document text
+		$fielddata=~s@<@&lt;@g;
+		$fielddata=~s@>@&gt;@g; # for xml protection...
+		$fielddata=~s@_@\\_@g; # for macro language protection...
+		$doc_obj->add_utf8_text($cursection, $fielddata);
+	    } elsif ($fieldname eq "Identifier") {
+		# use as greenstone's unique record id
+		if ($fielddata =~ /^\d+$/) {
+		    # don't allow IDs that are completely numeric
+		    $unique_id="id" . $fielddata;
+		} else {
+		    $unique_id=$fielddata;
+		}
+	    } else {
+		# add as document metadata
+		$fielddata=~s/\[/&#91;/g;
+		$fielddata=~s/\]/&#93;/g;
+		$doc_obj->add_utf8_metadata($cursection,
+					    $fieldname, $fielddata);
+
+	    }
+	}
+
+	if (!defined $unique_id) {
+	    $doc_obj->set_OID($hash . "s$count");
+	} else {
+	    # use our id from the database...
+	    $doc_obj->set_OID($unique_id);
+	}
+
+
+        # process the document
+	$processor->process($doc_obj);
+
+
+	$count++;
+
+	# get next row
+	@row_array=$statement_hand->fetchrow_array; # fetchrow_hashref?
+    } # end of row_array is not empty
+
+    # check "$sth->err" if empty array for error
+    if ($statement_hand->err) {
+	print "received error: \"" . $statement_hand->errstr . "\"\n";
+    }
+
+    # clean up connection to database
+    $statement_hand->finish();
+    $dbhandle->disconnect();
+
+    # num of input files, rather than documents created?
+    $self->{'num_processed'}++;
+
+    return $count;
+}
+
+1;
