Index: in/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm	(revision 32789)
+++ 	(revision )
@@ -1,464 +1,0 @@
-###########################################################################
-#
-# GreenstoneSQLPlugin.pm -- reads into doc_obj from SQL db and docsql.xml
-# Metadata and/or fulltext are stored in SQL db, the rest may be stored in
-# the docsql .xml files.
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package GreenstoneSQLPlugin;
-
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use DBI;
-use docprint; # for new unescape_text() subroutine
-use GreenstoneXMLPlugin;
-use gsmysql;
-
-#use unicode;
-#use Encode;
-
-########################################################################################
-
-# GreenstoneSQLPlugin inherits from GreenstoneXMLPlugin so that if meta or fulltext
-# is still written out to doc.xml (docsql .xml), that will be processed as usual,
-# whereas GreenstoneSQLPlugin will process all the rest (full text and/or meta, whichever
-# is written out by GreenstoneSQLPlugout into the SQL db).
-
-
-sub BEGIN {
-    @GreenstoneSQLPlugin::ISA = ('GreenstoneXMLPlugin');
-}
-
-# This plugin must be in the document plugins pipeline IN PLACE OF GreenstoneXMLPlugin
-# So we won't have a process exp conflict here.
-# The structure of docsql.xml files is identical to doc.xml and the contents are similar except:
-#   - since metadata and/or fulltxt are stored in mysql db instead, just XML comments indicating
-#   this are left inside docsql.xml within the <Description> (for meta) and/or <Content> (for txt)
-#   - the root element Archive now has a docoid attribute: <Archive docoid="OID">
-sub get_default_process_exp {
-    my $self = shift (@_);
-
-    return q^(?i)docsql(-\d+)?\.xml$^; # regex based on this method in GreenstoneXMLPlugin
-}
-
-my $process_mode_list =
-    [ { 'name' => "meta_only",
-        'desc' => "{GreenstoneSQLPlug.process_mode.meta_only}" },      
-      { 'name' => "text_only",
-        'desc' => "{GreenstoneSQLPlug.process_mode.text_only}" },
-      { 'name' => "all",
-        'desc' => "{GreenstoneSQLPlug.process_mode.all}" } ];
-
-my $rollback_on_cancel_list =
-    [ { 'name' => "true",
-        'desc' => "{common.true}" },      
-      { 'name' => "false",
-        'desc' => "{common.false}" } ];
-
-# NOTE: If subclassing gsmysql for other supporting databases and if they have different required
-# connection parameters, we can check how WordPlugin, upon detecting Word is installed,
-# dynamically loads Word specific configuration options.
-my $arguments =
-    [ { 'name' => "process_exp",
-	'desc' => "{BaseImporter.process_exp}",
-	'type' => "regexp",
-	'deft' => &get_default_process_exp(),
-	'reqd' => "no" },
-      { 'name' => "process_mode", 
-	'desc' => "{GreenstoneSQLPlug.process_mode}",
-	'type' => "enum",
-	'list' => $process_mode_list,
-	'deft' => "all",
-	'reqd' => "no"},
-      { 'name' => "rollback_on_cancel", 
-	'desc' => "{GreenstoneSQLPlug.rollback_on_cancel}",
-	'type' => "enum",
-	'list' => $rollback_on_cancel_list,
-	'deft' => "false", # better default than true
-	'reqd' => "no",
-	'hiddengli' => "no"},
-      { 'name' => "db_driver", 
-	'desc' => "{GreenstoneSQLPlug.db_driver}",
-	'type' => "string",	
-	'deft' => "mysql",
-	'reqd' => "yes"},
-      { 'name' => "db_client_user", 
-	'desc' => "{GreenstoneSQLPlug.db_client_user}",
-	'type' => "string",	
-	'deft' => "root",
-	'reqd' => "yes"},
-      { 'name' => "db_client_pwd", 
-	'desc' => "{GreenstoneSQLPlug.db_client_pwd}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "no"}, # pwd not required: can create mysql accounts without pwd
-      { 'name' => "db_host", 
-	'desc' => "{GreenstoneSQLPlug.db_host}",
-	'type' => "string",
-	'deft' => "127.0.0.1", # NOTE: make this int? No default for port, since it's not a required connection param
-	'reqd' => "yes"},
-      { 'name' => "db_port", 
-	'desc' => "{GreenstoneSQLPlug.db_port}",
-	'type' => "string", # NOTE: make this int? No default for port, since it's not a required connection param
-	'reqd' => "no"}
-    ];
-
-my $options = { 'name'     => "GreenstoneSQLPlugin",
-		'desc'     => "{GreenstoneSQLPlugin.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-	        'args'     => $arguments };
-
-
-###### Methods called during buildcol and import #######
-
-sub new {
-    my ($class) = shift (@_);
-    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
-    push(@$pluginlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new GreenstoneXMLPlugin($pluginlist, $inputargs, $hashArgOptLists);
-
-    
-    #return bless $self, $class;
-    $self = bless $self, $class;
-    if ($self->{'info_only'}) {
-	# If running pluginfo, we don't need to go further.
-	return $self;
-    }
-
-    # do anything else that needs to be done here when not pluginfo
-    
-    return $self;
-}
-
-# Call init() not begin() because there can be multiple plugin passes and begin() called for
-# each pass (one for doc level and another for section level indexing), whereas init() should
-# be called before any and all passes.
-# This way, we can connect to the SQL database once per buildcol run.
-# Although now it doesn't matter, since gsmysql.pm uses the get_instance pattern to return a
-# singleton db connection, regardless of the number of gsmysql objects instantiated and
-# the number of connect() calls made on them.
-sub init {
-    my ($self) = shift (@_);
-    
-    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
-
-
-    # create gsmysql object.
-    # collection name will be used for naming tables (site name will be used for naming database)
-    my $gs_sql = new gsmysql({
-	'collection_name' => $ENV{'GSDLCOLLECTION'},
-	'verbosity' => $self->{'verbosity'} || 0
-			   });
-    
-    # if autocommit is set, there's no rollback support
-    my $autocommit = ($self->{'rollback_on_cancel'} eq "false") ? 1 : 0;
-
-    # try connecting to the mysql db, die if that fails
-    if(!$gs_sql->connect_to_db({
-	'db_driver' => $self->{'db_driver'},
-	'db_client_user' => $self->{'db_client_user'},
-	'db_client_pwd' => $self->{'db_client_pwd'},
-	'db_host' => $self->{'db_host'},
-	'db_port' => $self->{'db_port'}, # undef by default, can leave as is
-	'autocommit' => $autocommit
-			       })
-	)
-    {
-	# This is fatal for the plugout, let's terminate here
-	# PrintError would already have displayed the warning message on connection fail	
-	die("Could not connect to db. Can't proceed.\n");
-    }
-    
-    my $db_name = $self->{'site'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
-
-    # Attempt to use the db, create it if it doesn't exist (but don't create the tables yet)
-    # Bail if we can't use the database
-    if(!$gs_sql->use_db($db_name)) {
-	
-	# This is fatal for the plugout, let's terminate here after disconnecting again
-	# PrintError would already have displayed the warning message on load fail
-	# And on die() perl will call gsmysql destroy which will ensure a disconnect() from db
-	die("Could not use db $db_name. Can't proceed.\n");
-    }
-    
-    
-    # store db handle now that we're connected
-    $self->{'gs_sql'} = $gs_sql;    
-}
-
-
-# This method also runs on import.pl if gs_sql has a value.
-# Call deinit() not end() because there can be multiple plugin passes:
-# one for doc level and another for section level indexing
-# and deinit() should be called before all passes
-# This way, we can close the SQL database once per buildcol run.
-# Again, this doesn't matter because we gsmysql the ensures the connection
-# is a singleton connection instance, which connects once and disconnects once per perl process.
-sub deinit {
-    my ($self) = shift (@_);
-    
-    if($self->{'gs_sql'}) {
-
-	# Important to call finished():
-	# it will disconnect from db if this is the last gsmysql instance,
-	# and it will commit to db before disconnecting if rollbback_on_cancel turned on
-	$self->{'gs_sql'}->finished();
-
-	# Clear gs_sql (setting key to undef has a different meaning from deleting:
-	# undef makes key still exist but its value is unded whereas delete deletes the key)
-	# So all future use has to make the connection again
-	delete $self->{'gs_sql'};
-    }
-
-    $self->SUPER::deinit(@_);
-}
-
-
-
-###### Methods only called during import.pl #####
-
-# This is called once if removeold is set with import.pl. Most plugins will do
-# nothing but if a plugin does any stuff outside of creating doc obj, then 
-# it may need to clear something.
-# In the case of GreenstoneSQL plugs: this is the first time we have a chance
-# to purge the tables of the current collection from the current site's database
-sub remove_all {
-    my $self = shift (@_);
-    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
-
-    $self->SUPER::remove_all(@_);
-    
-    print STDERR "   Building with removeold option set, so deleting current collection's tables if they exist\n" if($self->{'verbosity'});
-    
-    # if we're in here, we'd already have run 'use database <site>;' during sub init()
-    # so we can go ahead and delete the collection's tables
-    my $gs_sql = $self->{'gs_sql'};
-    $gs_sql->delete_collection_tables(); # will delete them if they exist
-
-    # and recreate tables? No. Tables' existence is ensured in GreenstoneSQLPlugout::begin()
-    my $proc_mode = $self->{'process_mode'};
-    if($proc_mode ne "text_only") {
-	$gs_sql->ensure_meta_table_exists();
-    }
-    if($proc_mode ne "meta_only") {
-	$gs_sql->ensure_fulltxt_table_exists();
-    }
-
-}
-
-# This is called during import.pl per document for docs that have been deleted from the 
-# collection. Most plugins will do nothing
-# but if a plugin does any stuff outside of creating doc obj, then it may need
-# to clear something.
-# In the case of GreenstoneSQL plugs: Remove the doc(s) denoted by oids from GS SQL db.
-# This takes care of incremental deletes (docs marked D by ArchivesInfPlugin when building
-# incrementally) as well as cases of "Non-icremental Delete", see ArchivesInfPlugin.pm.
-# As well as cases involving reindexing, which are implemented here as delete followed by add.
-sub remove_one {
-    my $self = shift (@_);
-    
-    my ($file, $oids, $archivedir) = @_;
-
-    my $rv = $self->SUPER::remove_one(@_);
-    
-    print STDERR "@@@ IN SQLPLUG::REMOVE_ONE: $file\n";
-    
-    #return undef unless $self->can_process_this_file($file); # NO, DON'T DO THIS (inherited remove_one behaviour) HERE: 
-           # WE DON'T CARE IF IT'S AN IMAGE FILE THAT WAS DELETED.
-           # WE CARE ABOUT REMOVING THE DOC_OID OF THAT IMAGE FILE FROM THE SQL DB
-           # SO DON'T RETURN IF CAN'T_PROCESS_THIS_FILE
-    
-    
-    my $gs_sql = $self->{'gs_sql'} || return 0; # couldn't make the connection or no db etc
-    
-    my $proc_mode = $self->{'process_mode'};
-    foreach my $oid (@$oids) {	
-	if($proc_mode eq "all" || $proc_mode eq "meta_only") {
-	    print STDERR "@@@@@@@@ Deleting $oid from meta table\n" if $self->{'verbosity'} > 2;
-	    $gs_sql->delete_recs_from_metatable_with_docid($oid);
-	}
-	if($proc_mode eq "all" || $proc_mode eq "text_only") {
-	    print STDERR "@@@@@@@@ Deleting $oid from fulltxt table\n" if $self->{'verbosity'} > 2;
-	    $gs_sql->delete_recs_from_texttable_with_docid($oid);
-	}
-    }
-    return $rv;
-}
-
-##### Methods called only during buildcol #####
-
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    
-    $self->{'element'} = $element;
-    if ($element eq "Archive") { # docsql.xml files contain a OID attribute on Archive element
-	# the element's attributes are in %_ as per ReadXMLFile::xml_start_tag() (while $_
-	# contains the tag)
-
-	# Don't access %_{'docoid'} directly: keep getting a warning message to 
-	# use $_{'docoid'} for scalar contexts, but %_ is the element's attr hashmap 
-	# whereas $_ has the tag info. So we don't want to do $_{'docoid'}.
-	my %attr_hash = %_; # right way, see OAIPlugin.pm
-	$self->{'doc_oid'} = $attr_hash{'docoid'};
-	print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
-	    if $self->{'verbosity'} > 2;
-
-    }
-    else { # let superclass GreenstoneXMLPlugin continue to process <Section> and <Metadata> elements
-	$self->SUPER::xml_start_tag(@_);
-    }
-}
-
-# There are multiple passes processing the document (see buildcol's mode parameter description):
-# - compressing the text which may be a dummy pass for lucene/solr, wherein they still want the
-# docobj for different purposes,
-# - the pass(es) for indexing, e.g. doc/didx and section/sidx level passes
-# - and an infodb pass for processing the classifiers. This pass too needs the docobj
-# Since all passes need the doc_obj, all are read in from docsql + SQL db into the docobj in memory
-
-# We only ever get here or do any parsing of the docsql.xml file during the buildcol.pl phase.
-# At the end of superclass GreenstoneXMLPlugin.pm's close_document() method,
-# the doc_obj in memory is processed (indexed) and then made undef.
-# So we have to work with doc_obj before superclass close_document() is finished.
-sub close_document {
-    my $self = shift(@_);
-
-    my $gs_sql = $self->{'gs_sql'};
-    
-    my $outhandle = $self->{'outhandle'};
-    my $doc_obj = $self->{'doc_obj'};
-
-    my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
-    my $proc_mode = $self->{'process_mode'};
-    
-    # For now, we have access to doc_obj (until just before super::close_document() terminates)
-
-    # OID parsed of docsql.xml file does need to be set on $doc_obj, as noticed in this case:
-    # when a doc in import is renamed, and you do incremental import, it is marked for reindexing 
-    # (reindexing is implemented by this plugin as a delete followed by add into the sql db).
-    # In that case, UNLESS you set the OID at this stage, the old deleted doc id (for the old doc
-    # name) continues to exist in the index at the end of incremental rebuilding if you were to
-    # browse the rebuilt collection by files/titles. So unless you set the OID here, the deleted
-    # doc oids will still be listed in the index.
-	
-    print STDERR "   GreenstoneSQLPlugin processing doc $oid (reading into docobj from SQL db)\n"
-    	if $self->{'verbosity'};
-    
-    my $build_proc_mode = $self->{'processor'}->get_mode(); # can be "text" as per basebuildproc or infodb
-    if($build_proc_mode =~ m/(delete)$/) {
-	# build_proc_mode could be "(infodb|text)(delete|reindex)"
-	# "...delete" or "...reindex" as per ArchivesInfPlugin
-	# But reindex is implemented as delete for GreenstoneSQLPlugs, so that's all we see here?
-	print STDERR "   DOC $oid WAS MARKED FOR DELETION. Won't attempt to retrieve from SQL db.\n" if $self->{'verbosity'};
-	$self->{'doc_obj'}->set_OID($oid); # oid is all we care about for a doc marked for deletion
-	$self->SUPER::close_document(@_); # at the end of this method, doc will have been deleted
-	return;	 # oid of doc marked for deletion is not in the SQL db, don't bother looking it up
-    }
-
-    # else, doc denoted by oid was not marked for deletion, look up its oid in db and read it into doc obj
-    
-    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
-	# read in meta for the collection (i.e. select * from <col>_metadata table
-	
-	my $records = $gs_sql->select_from_metatable_matching_docid($oid, $outhandle);
-	
-	print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
-
-	foreach my $row (@$records) {
-	    my ($primary_key, $did, $sid, $metaname, $metaval) = @$row;
-		
-	    # get rid of the artificial "root" introduced in section id when saving to sql db
-	    $sid =~ s@^root@@;
-	    $sid = $doc_obj->get_top_section() unless $sid;
-	    print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
-		if $self->{'verbosity'} > 2;
-	    
-	    # We're only dealing with utf8 data where docobj is concerned
-	    # Data stored unescaped in db: escaping only for html/xml files, not for txt files or db
-	    $doc_obj->add_utf8_metadata($sid, $metaname, $metaval);
-	}
-	
-	# UTF8 debugging, e.g. if we have a with macron in dc.Title
-	##binmode(STDERR, ":utf8"); # not necessary when calling utf8decomp to print wide characters as unicode codept, avoiding 'wide character' warnings.
-	#my $dcTitle = $doc_obj->get_metadata_element($doc_obj->get_top_section(), "dc.Title");	
-	#print STDERR "###### dc.Title: ".&unicode::utf8decomp($dcTitle)."\n";
-	#print STDERR "###### DECODED dc.Title: ".&unicode::utf8decomp(&Encode::decode_utf8($dcTitle))."\n"; # decoding was needed for perl 5.18
-	
-	print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
-	    if $self->{'verbosity'} > 2;
-    }
-	
-    if($proc_mode eq "all" || $proc_mode eq "text_only") {
-	# read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
-	
-	my $fulltxt_table = $gs_sql->get_fulltext_table_name();
-	
-	
-	my $records = $gs_sql->select_from_texttable_matching_docid($oid, $outhandle);
-	
-	
-	print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
-	    if $self->{'verbosity'} > 2;
-
-	foreach my $row (@$records) {
-	    my ($primary_key, $did, $sid, $text) = @$row;
-	    
-	    # get rid of the artificial "root" introduced in section id when saving to sql db
-	    #$sid =~ s@^root@@;
-	    $sid = $doc_obj->get_top_section() if ($sid eq "root");
-	    print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
-		if $self->{'verbosity'} > 2;
-
-	    # We're only dealing with utf8 data where docobj is concerned
-	    # Data stored unescaped in db: escaping is only for html/xml files, not for txt files or db
-	    $doc_obj->add_utf8_textref($sid, \$text);
-	}	
-	print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
-	    if $self->{'verbosity'} > 2;
-    }
-    
-
-    # setting OID here instead of before reading from SQL db into docobj, will prevent duplicate values for Identifier
-    # since doc::set_OID() calls doc::set_metadata_element() for metadata that can't occur more than once
-    $self->{'doc_obj'}->set_OID($oid); # may only be necessary if doc was marked for deletion so that SUPER::close_document knows
-                                       # the oid of marked doc to remove from index
-    
-    # done reading into docobj from SQL db
-	
-    # don't forget to clean up on close() in superclass
-    # It will get the doc_obj indexed then make it undef
-    $self->SUPER::close_document(@_);
-}
-
-
-1;
Index: /main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm.inactive
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm.inactive	(revision 32790)
+++ /main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm.inactive	(revision 32790)
@@ -0,0 +1,464 @@
+###########################################################################
+#
+# GreenstoneSQLPlugin.pm -- reads into doc_obj from SQL db and docsql.xml
+# Metadata and/or fulltext are stored in SQL db, the rest may be stored in
+# the docsql .xml files.
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package GreenstoneSQLPlugin;
+
+
+use strict;
+no strict 'refs'; # allow filehandles to be variables and viceversa
+
+use DBI;
+use docprint; # for new unescape_text() subroutine
+use GreenstoneXMLPlugin;
+use gsmysql;
+
+#use unicode;
+#use Encode;
+
+########################################################################################
+
+# GreenstoneSQLPlugin inherits from GreenstoneXMLPlugin so that if meta or fulltext
+# is still written out to doc.xml (docsql .xml), that will be processed as usual,
+# whereas GreenstoneSQLPlugin will process all the rest (full text and/or meta, whichever
+# is written out by GreenstoneSQLPlugout into the SQL db).
+
+
+sub BEGIN {
+    @GreenstoneSQLPlugin::ISA = ('GreenstoneXMLPlugin');
+}
+
+# This plugin must be in the document plugins pipeline IN PLACE OF GreenstoneXMLPlugin
+# So we won't have a process exp conflict here.
+# The structure of docsql.xml files is identical to doc.xml and the contents are similar except:
+#   - since metadata and/or fulltxt are stored in mysql db instead, just XML comments indicating
+#   this are left inside docsql.xml within the <Description> (for meta) and/or <Content> (for txt)
+#   - the root element Archive now has a docoid attribute: <Archive docoid="OID">
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)docsql(-\d+)?\.xml$^; # regex based on this method in GreenstoneXMLPlugin
+}
+
+my $process_mode_list =
+    [ { 'name' => "meta_only",
+        'desc' => "{GreenstoneSQLPlug.process_mode.meta_only}" },      
+      { 'name' => "text_only",
+        'desc' => "{GreenstoneSQLPlug.process_mode.text_only}" },
+      { 'name' => "all",
+        'desc' => "{GreenstoneSQLPlug.process_mode.all}" } ];
+
+my $rollback_on_cancel_list =
+    [ { 'name' => "true",
+        'desc' => "{common.true}" },      
+      { 'name' => "false",
+        'desc' => "{common.false}" } ];
+
+# NOTE: If subclassing gsmysql for other supporting databases and if they have different required
+# connection parameters, we can check how WordPlugin, upon detecting Word is installed,
+# dynamically loads Word specific configuration options.
+my $arguments =
+    [ { 'name' => "process_exp",
+	'desc' => "{BaseImporter.process_exp}",
+	'type' => "regexp",
+	'deft' => &get_default_process_exp(),
+	'reqd' => "no" },
+      { 'name' => "process_mode", 
+	'desc' => "{GreenstoneSQLPlug.process_mode}",
+	'type' => "enum",
+	'list' => $process_mode_list,
+	'deft' => "all",
+	'reqd' => "no"},
+      { 'name' => "rollback_on_cancel", 
+	'desc' => "{GreenstoneSQLPlug.rollback_on_cancel}",
+	'type' => "enum",
+	'list' => $rollback_on_cancel_list,
+	'deft' => "false", # better default than true
+	'reqd' => "no",
+	'hiddengli' => "no"},
+      { 'name' => "db_driver", 
+	'desc' => "{GreenstoneSQLPlug.db_driver}",
+	'type' => "string",	
+	'deft' => "mysql",
+	'reqd' => "yes"},
+      { 'name' => "db_client_user", 
+	'desc' => "{GreenstoneSQLPlug.db_client_user}",
+	'type' => "string",	
+	'deft' => "root",
+	'reqd' => "yes"},
+      { 'name' => "db_client_pwd", 
+	'desc' => "{GreenstoneSQLPlug.db_client_pwd}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "no"}, # pwd not required: can create mysql accounts without pwd
+      { 'name' => "db_host", 
+	'desc' => "{GreenstoneSQLPlug.db_host}",
+	'type' => "string",
+	'deft' => "127.0.0.1", # NOTE: make this int? No default for port, since it's not a required connection param
+	'reqd' => "yes"},
+      { 'name' => "db_port", 
+	'desc' => "{GreenstoneSQLPlug.db_port}",
+	'type' => "string", # NOTE: make this int? No default for port, since it's not a required connection param
+	'reqd' => "no"}
+    ];
+
+my $options = { 'name'     => "GreenstoneSQLPlugin",
+		'desc'     => "{GreenstoneSQLPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+	        'args'     => $arguments };
+
+
+###### Methods called during buildcol and import #######
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    my $self = new GreenstoneXMLPlugin($pluginlist, $inputargs, $hashArgOptLists);
+
+    
+    #return bless $self, $class;
+    $self = bless $self, $class;
+    if ($self->{'info_only'}) {
+	# If running pluginfo, we don't need to go further.
+	return $self;
+    }
+
+    # do anything else that needs to be done here when not pluginfo
+    
+    return $self;
+}
+
+# Call init() not begin() because there can be multiple plugin passes and begin() called for
+# each pass (one for doc level and another for section level indexing), whereas init() should
+# be called before any and all passes.
+# This way, we can connect to the SQL database once per buildcol run.
+# Although now it doesn't matter, since gsmysql.pm uses the get_instance pattern to return a
+# singleton db connection, regardless of the number of gsmysql objects instantiated and
+# the number of connect() calls made on them.
+sub init {
+    my ($self) = shift (@_);
+    
+    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
+
+
+    # create gsmysql object.
+    # collection name will be used for naming tables (site name will be used for naming database)
+    my $gs_sql = new gsmysql({
+	'collection_name' => $ENV{'GSDLCOLLECTION'},
+	'verbosity' => $self->{'verbosity'} || 0
+			   });
+    
+    # if autocommit is set, there's no rollback support
+    my $autocommit = ($self->{'rollback_on_cancel'} eq "false") ? 1 : 0;
+
+    # try connecting to the mysql db, die if that fails
+    if(!$gs_sql->connect_to_db({
+	'db_driver' => $self->{'db_driver'},
+	'db_client_user' => $self->{'db_client_user'},
+	'db_client_pwd' => $self->{'db_client_pwd'},
+	'db_host' => $self->{'db_host'},
+	'db_port' => $self->{'db_port'}, # undef by default, can leave as is
+	'autocommit' => $autocommit
+			       })
+	)
+    {
+	# This is fatal for the plugout, let's terminate here
+	# PrintError would already have displayed the warning message on connection fail	
+	die("Could not connect to db. Can't proceed.\n");
+    }
+    
+    my $db_name = $self->{'site'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
+
+    # Attempt to use the db, create it if it doesn't exist (but don't create the tables yet)
+    # Bail if we can't use the database
+    if(!$gs_sql->use_db($db_name)) {
+	
+	# This is fatal for the plugout, let's terminate here after disconnecting again
+	# PrintError would already have displayed the warning message on load fail
+	# And on die() perl will call gsmysql destroy which will ensure a disconnect() from db
+	die("Could not use db $db_name. Can't proceed.\n");
+    }
+    
+    
+    # store db handle now that we're connected
+    $self->{'gs_sql'} = $gs_sql;    
+}
+
+
+# This method also runs on import.pl if gs_sql has a value.
+# Call deinit() not end() because there can be multiple plugin passes:
+# one for doc level and another for section level indexing
+# and deinit() should be called before all passes
+# This way, we can close the SQL database once per buildcol run.
+# Again, this doesn't matter because we gsmysql the ensures the connection
+# is a singleton connection instance, which connects once and disconnects once per perl process.
+sub deinit {
+    my ($self) = shift (@_);
+    
+    if($self->{'gs_sql'}) {
+
+	# Important to call finished():
+	# it will disconnect from db if this is the last gsmysql instance,
+	# and it will commit to db before disconnecting if rollbback_on_cancel turned on
+	$self->{'gs_sql'}->finished();
+
+	# Clear gs_sql (setting key to undef has a different meaning from deleting:
+	# undef makes key still exist but its value is unded whereas delete deletes the key)
+	# So all future use has to make the connection again
+	delete $self->{'gs_sql'};
+    }
+
+    $self->SUPER::deinit(@_);
+}
+
+
+
+###### Methods only called during import.pl #####
+
+# This is called once if removeold is set with import.pl. Most plugins will do
+# nothing but if a plugin does any stuff outside of creating doc obj, then 
+# it may need to clear something.
+# In the case of GreenstoneSQL plugs: this is the first time we have a chance
+# to purge the tables of the current collection from the current site's database
+sub remove_all {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+
+    $self->SUPER::remove_all(@_);
+    
+    print STDERR "   Building with removeold option set, so deleting current collection's tables if they exist\n" if($self->{'verbosity'});
+    
+    # if we're in here, we'd already have run 'use database <site>;' during sub init()
+    # so we can go ahead and delete the collection's tables
+    my $gs_sql = $self->{'gs_sql'};
+    $gs_sql->delete_collection_tables(); # will delete them if they exist
+
+    # and recreate tables? No. Tables' existence is ensured in GreenstoneSQLPlugout::begin()
+    my $proc_mode = $self->{'process_mode'};
+    if($proc_mode ne "text_only") {
+	$gs_sql->ensure_meta_table_exists();
+    }
+    if($proc_mode ne "meta_only") {
+	$gs_sql->ensure_fulltxt_table_exists();
+    }
+
+}
+
+# This is called during import.pl per document for docs that have been deleted from the 
+# collection. Most plugins will do nothing
+# but if a plugin does any stuff outside of creating doc obj, then it may need
+# to clear something.
+# In the case of GreenstoneSQL plugs: Remove the doc(s) denoted by oids from GS SQL db.
+# This takes care of incremental deletes (docs marked D by ArchivesInfPlugin when building
+# incrementally) as well as cases of "Non-icremental Delete", see ArchivesInfPlugin.pm.
+# As well as cases involving reindexing, which are implemented here as delete followed by add.
+sub remove_one {
+    my $self = shift (@_);
+    
+    my ($file, $oids, $archivedir) = @_;
+
+    my $rv = $self->SUPER::remove_one(@_);
+    
+    print STDERR "@@@ IN SQLPLUG::REMOVE_ONE: $file\n";
+    
+    #return undef unless $self->can_process_this_file($file); # NO, DON'T DO THIS (inherited remove_one behaviour) HERE: 
+           # WE DON'T CARE IF IT'S AN IMAGE FILE THAT WAS DELETED.
+           # WE CARE ABOUT REMOVING THE DOC_OID OF THAT IMAGE FILE FROM THE SQL DB
+           # SO DON'T RETURN IF CAN'T_PROCESS_THIS_FILE
+    
+    
+    my $gs_sql = $self->{'gs_sql'} || return 0; # couldn't make the connection or no db etc
+    
+    my $proc_mode = $self->{'process_mode'};
+    foreach my $oid (@$oids) {	
+	if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+	    print STDERR "@@@@@@@@ Deleting $oid from meta table\n" if $self->{'verbosity'} > 2;
+	    $gs_sql->delete_recs_from_metatable_with_docid($oid);
+	}
+	if($proc_mode eq "all" || $proc_mode eq "text_only") {
+	    print STDERR "@@@@@@@@ Deleting $oid from fulltxt table\n" if $self->{'verbosity'} > 2;
+	    $gs_sql->delete_recs_from_texttable_with_docid($oid);
+	}
+    }
+    return $rv;
+}
+
+##### Methods called only during buildcol #####
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    
+    $self->{'element'} = $element;
+    if ($element eq "Archive") { # docsql.xml files contain a OID attribute on Archive element
+	# the element's attributes are in %_ as per ReadXMLFile::xml_start_tag() (while $_
+	# contains the tag)
+
+	# Don't access %_{'docoid'} directly: keep getting a warning message to 
+	# use $_{'docoid'} for scalar contexts, but %_ is the element's attr hashmap 
+	# whereas $_ has the tag info. So we don't want to do $_{'docoid'}.
+	my %attr_hash = %_; # right way, see OAIPlugin.pm
+	$self->{'doc_oid'} = $attr_hash{'docoid'};
+	print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
+	    if $self->{'verbosity'} > 2;
+
+    }
+    else { # let superclass GreenstoneXMLPlugin continue to process <Section> and <Metadata> elements
+	$self->SUPER::xml_start_tag(@_);
+    }
+}
+
+# There are multiple passes processing the document (see buildcol's mode parameter description):
+# - compressing the text which may be a dummy pass for lucene/solr, wherein they still want the
+# docobj for different purposes,
+# - the pass(es) for indexing, e.g. doc/didx and section/sidx level passes
+# - and an infodb pass for processing the classifiers. This pass too needs the docobj
+# Since all passes need the doc_obj, all are read in from docsql + SQL db into the docobj in memory
+
+# We only ever get here or do any parsing of the docsql.xml file during the buildcol.pl phase.
+# At the end of superclass GreenstoneXMLPlugin.pm's close_document() method,
+# the doc_obj in memory is processed (indexed) and then made undef.
+# So we have to work with doc_obj before superclass close_document() is finished.
+sub close_document {
+    my $self = shift(@_);
+
+    my $gs_sql = $self->{'gs_sql'};
+    
+    my $outhandle = $self->{'outhandle'};
+    my $doc_obj = $self->{'doc_obj'};
+
+    my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
+    my $proc_mode = $self->{'process_mode'};
+    
+    # For now, we have access to doc_obj (until just before super::close_document() terminates)
+
+    # OID parsed of docsql.xml file does need to be set on $doc_obj, as noticed in this case:
+    # when a doc in import is renamed, and you do incremental import, it is marked for reindexing 
+    # (reindexing is implemented by this plugin as a delete followed by add into the sql db).
+    # In that case, UNLESS you set the OID at this stage, the old deleted doc id (for the old doc
+    # name) continues to exist in the index at the end of incremental rebuilding if you were to
+    # browse the rebuilt collection by files/titles. So unless you set the OID here, the deleted
+    # doc oids will still be listed in the index.
+	
+    print STDERR "   GreenstoneSQLPlugin processing doc $oid (reading into docobj from SQL db)\n"
+    	if $self->{'verbosity'};
+    
+    my $build_proc_mode = $self->{'processor'}->get_mode(); # can be "text" as per basebuildproc or infodb
+    if($build_proc_mode =~ m/(delete)$/) {
+	# build_proc_mode could be "(infodb|text)(delete|reindex)"
+	# "...delete" or "...reindex" as per ArchivesInfPlugin
+	# But reindex is implemented as delete for GreenstoneSQLPlugs, so that's all we see here?
+	print STDERR "   DOC $oid WAS MARKED FOR DELETION. Won't attempt to retrieve from SQL db.\n" if $self->{'verbosity'};
+	$self->{'doc_obj'}->set_OID($oid); # oid is all we care about for a doc marked for deletion
+	$self->SUPER::close_document(@_); # at the end of this method, doc will have been deleted
+	return;	 # oid of doc marked for deletion is not in the SQL db, don't bother looking it up
+    }
+
+    # else, doc denoted by oid was not marked for deletion, look up its oid in db and read it into doc obj
+    
+    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+	# read in meta for the collection (i.e. select * from <col>_metadata table
+	
+	my $records = $gs_sql->select_from_metatable_matching_docid($oid, $outhandle);
+	
+	print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
+
+	foreach my $row (@$records) {
+	    my ($primary_key, $did, $sid, $metaname, $metaval) = @$row;
+		
+	    # get rid of the artificial "root" introduced in section id when saving to sql db
+	    $sid =~ s@^root@@;
+	    $sid = $doc_obj->get_top_section() unless $sid;
+	    print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
+		if $self->{'verbosity'} > 2;
+	    
+	    # We're only dealing with utf8 data where docobj is concerned
+	    # Data stored unescaped in db: escaping only for html/xml files, not for txt files or db
+	    $doc_obj->add_utf8_metadata($sid, $metaname, $metaval);
+	}
+	
+	# UTF8 debugging, e.g. if we have a with macron in dc.Title
+	##binmode(STDERR, ":utf8"); # not necessary when calling utf8decomp to print wide characters as unicode codept, avoiding 'wide character' warnings.
+	#my $dcTitle = $doc_obj->get_metadata_element($doc_obj->get_top_section(), "dc.Title");	
+	#print STDERR "###### dc.Title: ".&unicode::utf8decomp($dcTitle)."\n";
+	#print STDERR "###### DECODED dc.Title: ".&unicode::utf8decomp(&Encode::decode_utf8($dcTitle))."\n"; # decoding was needed for perl 5.18
+	
+	print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
+	    if $self->{'verbosity'} > 2;
+    }
+	
+    if($proc_mode eq "all" || $proc_mode eq "text_only") {
+	# read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
+	
+	my $fulltxt_table = $gs_sql->get_fulltext_table_name();
+	
+	
+	my $records = $gs_sql->select_from_texttable_matching_docid($oid, $outhandle);
+	
+	
+	print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
+	    if $self->{'verbosity'} > 2;
+
+	foreach my $row (@$records) {
+	    my ($primary_key, $did, $sid, $text) = @$row;
+	    
+	    # get rid of the artificial "root" introduced in section id when saving to sql db
+	    #$sid =~ s@^root@@;
+	    $sid = $doc_obj->get_top_section() if ($sid eq "root");
+	    print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
+		if $self->{'verbosity'} > 2;
+
+	    # We're only dealing with utf8 data where docobj is concerned
+	    # Data stored unescaped in db: escaping is only for html/xml files, not for txt files or db
+	    $doc_obj->add_utf8_textref($sid, \$text);
+	}	
+	print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
+	    if $self->{'verbosity'} > 2;
+    }
+    
+
+    # setting OID here instead of before reading from SQL db into docobj, will prevent duplicate values for Identifier
+    # since doc::set_OID() calls doc::set_metadata_element() for metadata that can't occur more than once
+    $self->{'doc_obj'}->set_OID($oid); # may only be necessary if doc was marked for deletion so that SUPER::close_document knows
+                                       # the oid of marked doc to remove from index
+    
+    # done reading into docobj from SQL db
+	
+    # don't forget to clean up on close() in superclass
+    # It will get the doc_obj indexed then make it undef
+    $self->SUPER::close_document(@_);
+}
+
+
+1;
Index: in/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm	(revision 32789)
+++ 	(revision )
@@ -1,350 +1,0 @@
-###########################################################################
-#
-# GreenstoneSQLPlugout.pm -- plugout module for writing all or some the 
-# Greenstone document format (metadata and/or fulltext) into a (My)SQL db.
-# The rest is then still written out by GreenstoneXMLPlugout as usual.
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package GreenstoneSQLPlugout;
-
-use strict;
-no strict 'refs';
-no strict 'subs';
-
-use GreenstoneXMLPlugout;
-use docprint;
-use gsmysql;
-
-use DBI; # the central package for this plugout
-
-#use unicode;
-
-# This plugout does not output the metadata and/or fulltxt xml to a file,
-# but outputs rows into a MySQL db table for metadata and/or a db table for fulltxt
-
-sub BEGIN {
-    @GreenstoneSQLPlugout::ISA = ('GreenstoneXMLPlugout');
-}
-
-
-my $process_mode_list =
-    [ { 'name' => "meta_only",
-        'desc' => "{GreenstoneSQLPlug.process_mode.meta_only}" },      
-      { 'name' => "text_only",
-        'desc' => "{GreenstoneSQLPlug.process_mode.text_only}" },
-      { 'name' => "all",
-        'desc' => "{GreenstoneSQLPlug.process_mode.all}" } ];
-
-my $rollback_on_cancel_list =
-    [ { 'name' => "true",
-        'desc' => "{common.true}" },      
-      { 'name' => "false",
-        'desc' => "{common.false}" } ];
-
-# The following are the saveas.options:
-my $arguments = [ 
-    { 'name' => "process_mode", 
-      'desc' => "{GreenstoneSQLPlug.process_mode}",
-      'type' => "enum",
-      'list' => $process_mode_list,
-      'deft' => "all",
-      'reqd' => "no",
-      'hiddengli' => "no"},
-    { 'name' => "rollback_on_cancel", 
-      'desc' => "{GreenstoneSQLPlug.rollback_on_cancel}",
-      'type' => "enum",
-      'list' => $rollback_on_cancel_list,
-      'deft' => "false", # better default than true
-      'reqd' => "no",
-      'hiddengli' => "no"},
-    { 'name' => "db_driver", 
-      'desc' => "{GreenstoneSQLPlug.db_driver}",
-      'type' => "string",	
-      'deft' => "mysql",
-      'reqd' => "yes"},
-    { 'name' => "db_client_user", 
-      'desc' => "{GreenstoneSQLPlug.db_client_user}",
-      'type' => "string",	
-      'deft' => "root",
-      'reqd' => "yes"},
-    { 'name' => "db_client_pwd", 
-      'desc' => "{GreenstoneSQLPlug.db_client_pwd}",
-      'type' => "string",
-      'deft' => "",
-      'reqd' => "no"}, # pwd not required: can create mysql accounts without pwd
-    { 'name' => "db_host", 
-      'desc' => "{GreenstoneSQLPlug.db_host}",
-      'type' => "string",
-      'deft' => "127.0.0.1", # localhost doesn't work for us, but 127.0.0.1 works. See gsmysql.pm
-      'reqd' => "yes"},
-    { 'name' => "db_port", 
-      'desc' => "{GreenstoneSQLPlug.db_port}",
-      'type' => "string", # NOTE: make this int? No default for port, since it's not a required connection param
-      'reqd' => "no"}
-    ];
-
-my $options = { 'name'     => "GreenstoneSQLPlugout",
-		'desc'     => "{GreenstoneSQLPlugout.desc}",
-		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
-
-##### This entire class is called only during import.pl #####
-
-##### Overridden methods #####
-
-sub new {
-    my ($class) = shift (@_);
-    my ($plugoutlist, $inputargs,$hashArgOptLists) = @_;
-    push(@$plugoutlist, $class);
-
-    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
-    push(@{$hashArgOptLists->{"OptList"}},$options);
-
-    my $self = new GreenstoneXMLPlugout($plugoutlist,$inputargs,$hashArgOptLists);
-    
-    if ($self->{'info_only'}) {	
-        # don't worry about any options etc
-        return bless $self, $class;
-    }
-    
-    return bless $self, $class;
-}
-
-# connect here and ensure all tables and databases exist
-sub begin {
-
-    my $self= shift (@_);
-    
-    my $db_params = {
-	'collection_name' => $ENV{'GSDLCOLLECTION'},
-	'verbosity' => $self->{'verbosity'} || 0
-	
-    };
-
-    my $gs_sql = new gsmysql($db_params);
-
-    # if autocommit is set, there's no rollback support
-    my $autocommit = ($self->{'rollback_on_cancel'} eq "false") ? 1 : 0;
-    
-    # try connecting to the mysql db, die if that fails
-    # So don't bother preparing GreenstoneXMLPlugout by calling superclass' begin() yet
-    if(!$gs_sql->connect_to_db({
-	'db_driver' => $self->{'db_driver'},
-	'db_client_user' => $self->{'db_client_user'},
-	'db_client_pwd' => $self->{'db_client_pwd'},
-	'db_host' => $self->{'db_host'},
-	'db_port' => $self->{'db_port'}, # undef by default, can leave as is
-	'autocommit' => $autocommit
-			       })
-	)
-    {
-	# This is fatal for the plugout, let's terminate here
-	# PrintError would already have displayed the warning message on connection fail	
-	die("Could not connect to db. Can't proceed.\n");
-    }
-
-    my $db_name = $self->{'site'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
-    my $proc_mode = $self->{'process_mode'};
-    
-
-    my $success = $gs_sql->use_db($db_name);
-    
-    if($success && $proc_mode ne "text_only") {
-	$success = $gs_sql->ensure_meta_table_exists();
-    }    
-    if($success && $proc_mode ne "meta_only") {
-	$success = $gs_sql->ensure_fulltxt_table_exists();
-    }
-    
-    if(!$success) {	
-	# This is fatal for the plugout, let's terminate here after disconnecting again
-	# PrintError would already have displayed the warning message on load fail
-	# And on die() perl will call gsmysql destroy which will ensure a disconnect() from db
-	die("Could not use db $db_name and/or prepare its tables. Can't proceed.\n");
-    }
-
-    # store the DBI wrapper instance
-    $self->{'gs_sql'} = $gs_sql;
-    
-    
-    # If setting up connection to sql db failed, we'd have terminated and wouldn't come up to here
-    # and wouldn't have bothered preparing GreenstoneXMLPlugout by calling superclass' begin().
-    # Finally, can call begin on super - important as doc.xml is opened as a group etc
-    
-    $self->SUPER::begin(@_);
-}
-
-# disconnect from database here, see inexport.pm
-sub end
-{
-    my $self = shift(@_);
-
-    # do the superclass stuff first, as any sql db failures should not prevent superclass cleanup
-    $self->SUPER::end(@_);    
-
-    # Important to call finished():
-    # it will disconnect from db if this is the last gsmysql instance,
-    # and it will commit to db before disconnecting if rollbback_on_cancel turned on
-    $self->{'gs_sql'}->finished();
-    delete $self->{'gs_sql'}; # key gs_sql no longer exists, not just the value being undef
-}
-
-# Produce files called docsql.xml instead of doc.xml
-sub get_doc_xml_filename {
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-    
-    return "docsql.xml";
-}
-
-# overriding to store doc OID as attribute of top level element: <Archive docoid="oid">
-sub output_xml_header {
-    my $self = shift (@_);
-    my ($outhandle, $doc_oid) = @_;
-
-    print $outhandle '<?xml version="1.0" encoding="utf-8" standalone="no"?>' . "\n";
-    print $outhandle "<!DOCTYPE Archive SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
-    print $outhandle "<Archive docoid=\"$doc_oid\">\n";
-}
- 
-# saveas() only generates the content in archives dir and in the SQL database
-sub saveas {
-    my $self = shift (@_);
-    my ($doc_obj, $doc_dir) = @_;
-
-    my $proc_mode = $self->{'process_mode'};
-    
-    # 1. pre save out and saving debug handle
-
-    # must call superclass (pre/post) saveas methods, as they handle assoc_files too
-    my ($docxml_outhandler, $output_file) = $self->SUPER::pre_saveas(@_);
-
-    $self->{'debug_outhandle'} = $docxml_outhandler if ($self->{'debug'}); # STDOUT if debug
-
-    # 2. overriding saving behaviour to do what the superclass does (writing out doc.xml files,
-    # under new name of docsql.xml, with breadcrumbs pointing to sql db) PLUS saving to sql db    
-
-    # NOTE: if proc_mode == all, then "breadcrumbs" (statements pointing viewer to the sql db
-    # for contents) go into both meta and txt elements of doc.xml (docsql.xml specifically): 
-	
-    # write the INVERSE into doc.xml as to what is written to the SQL db    
-    my $docxml_output_options = { 'output' => docprint::OUTPUT_NONE };
-    if($proc_mode eq "meta_only" ) { # since only meta to go into MySQL db, text will go into docxml
-	$docxml_output_options->{'output'} = docprint::OUTPUT_TEXT_ONLY;
-    } elsif($proc_mode eq "text_only" ) { # since only full text to go into MySQL db, meta will go into docxml
-	$docxml_output_options->{'output'} = docprint::OUTPUT_META_ONLY;
-    }
-    
-    # now we've prepared to write out whatever is meant to go into docxml
-    # and can do actual the steps superclass GreenstoneXMLPlugout carries out to write out docxml
-    # So: write out the doc xml file, "docsql.xml", for the current document
-    my $section_text = &docprint::get_section_xml($doc_obj, $docxml_output_options);
-    print $docxml_outhandler $section_text;    
-    
-    
-    # We also write out whatever needs to go into the MySQL database
-    $self->write_meta_and_text($doc_obj);
-
-    
-    # 3. post save out 
-    $self->SUPER::post_saveas($doc_obj, $doc_dir, $docxml_outhandler, $output_file);
-    
-    
-    # database connection is closed once, in end() method
-    # We're not opening and closing over and over for each doc during a single build
-}
-
-##### New methods, not inherited  #####
-
-# write meta and/or text PER DOC out to DB
-sub write_meta_and_text {
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-    my $doc_oid = $doc_obj->get_OID(); # this method processes a single doc at a time, so it uses the same OID throughout
-    my $root_section = $doc_obj->get_top_section();
-
-    $self->recursive_write_meta_and_text($doc_obj, $doc_oid, $root_section);
-}
-
-sub recursive_write_meta_and_text {
-    my $self = shift (@_);
-    my ($doc_obj, $doc_oid, $section) = @_;    
-
-    # If section=ROOT, write "root" as section name into table
-    # doc->get_top_section() is the name of the doc root section, which is ""
-    my $section_name = ($section eq "") ? "root" : $section;
-    
-    my $section_ptr = $doc_obj->_lookup_section ($section);
-    return "" unless defined $section_ptr;
-
-    my $debug_out = $self->{'debug_outhandle'};
-
-    my $gs_sql = $self->{'gs_sql'};
-    my $proc_mode = $self->{'process_mode'};
-    if($proc_mode eq "all" || $proc_mode eq "meta_only" ) {
-	
-	##binmode(STDERR, ":utf8"); # shouldn't be necessary as we call &unicode::utf8decomp() to avoid wide-character warnings by printing wide chars as unicode codepoints
-	#print STDERR "###### dc.Title: ".&unicode::utf8decomp($doc_obj->get_metadata_element($doc_obj->get_top_section(), "dc.Title"))."\n";
-	
-	foreach my $data (@{$section_ptr->{'metadata'}}) {
-	    my $meta_name = $data->[0];
-
-	    # Treat db like a text file instead of an html/xml file: don't need to escape text
-	    # going into it, unlike with doc(sql).xml
-	    my $meta_value = $data->[1];
-		
-	    # Write out the current section's meta to collection db's METADATA table	    
-	    
-	    # For each set of values to write to meta table, this next method call will
-	    # efficiently execute an insert SQL statement (using a prepared insert statement), 
-	    # filling in the values
-	    # OR if debugging, then it will print the SQL insert statement but not execute it
-	    # (a behaviour following what the GS XML Plugout superclass does on debug)
-	    
-	    $gs_sql->insert_row_into_metadata_table($doc_oid, $section_name, $meta_name, $meta_value, $self->{'debug'});
-	}
-    }
-    
-    
-    if($proc_mode eq "all" || $proc_mode eq "text_only" ) {
-
-	# See above, no need to html-escape for db
-	my $section_text = $section_ptr->{'text'};
-	
-	# fulltxt column can be SQL NULL. undef value for $section_text gets written out as NULL:
-	# https://stackoverflow.com/questions/12708633/which-one-represents-null-undef-or-empty-string
-	# The following will do the SQL insertion
-	# or if debug, the following will print the SQL insert stmt without executing it
-	$gs_sql->insert_row_into_fulltxt_table($doc_oid, $section_name, \$section_text, $self->{'debug'});
-    
-    }
-    
-    # output all subsections: RECURSIVE CALL
-    foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
-	$self->recursive_write_meta_and_text($doc_obj, $doc_oid, "$section.$subsection");
-    }
-}
-
-
-1;
Index: /main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm.inactive
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm.inactive	(revision 32790)
+++ /main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm.inactive	(revision 32790)
@@ -0,0 +1,350 @@
+###########################################################################
+#
+# GreenstoneSQLPlugout.pm -- plugout module for writing all or some the 
+# Greenstone document format (metadata and/or fulltext) into a (My)SQL db.
+# The rest is then still written out by GreenstoneXMLPlugout as usual.
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package GreenstoneSQLPlugout;
+
+use strict;
+no strict 'refs';
+no strict 'subs';
+
+use GreenstoneXMLPlugout;
+use docprint;
+use gsmysql;
+
+use DBI; # the central package for this plugout
+
+#use unicode;
+
+# This plugout does not output the metadata and/or fulltxt xml to a file,
+# but outputs rows into a MySQL db table for metadata and/or a db table for fulltxt
+
+sub BEGIN {
+    @GreenstoneSQLPlugout::ISA = ('GreenstoneXMLPlugout');
+}
+
+
+my $process_mode_list =
+    [ { 'name' => "meta_only",
+        'desc' => "{GreenstoneSQLPlug.process_mode.meta_only}" },      
+      { 'name' => "text_only",
+        'desc' => "{GreenstoneSQLPlug.process_mode.text_only}" },
+      { 'name' => "all",
+        'desc' => "{GreenstoneSQLPlug.process_mode.all}" } ];
+
+my $rollback_on_cancel_list =
+    [ { 'name' => "true",
+        'desc' => "{common.true}" },      
+      { 'name' => "false",
+        'desc' => "{common.false}" } ];
+
+# The following are the saveas.options:
+my $arguments = [ 
+    { 'name' => "process_mode", 
+      'desc' => "{GreenstoneSQLPlug.process_mode}",
+      'type' => "enum",
+      'list' => $process_mode_list,
+      'deft' => "all",
+      'reqd' => "no",
+      'hiddengli' => "no"},
+    { 'name' => "rollback_on_cancel", 
+      'desc' => "{GreenstoneSQLPlug.rollback_on_cancel}",
+      'type' => "enum",
+      'list' => $rollback_on_cancel_list,
+      'deft' => "false", # better default than true
+      'reqd' => "no",
+      'hiddengli' => "no"},
+    { 'name' => "db_driver", 
+      'desc' => "{GreenstoneSQLPlug.db_driver}",
+      'type' => "string",	
+      'deft' => "mysql",
+      'reqd' => "yes"},
+    { 'name' => "db_client_user", 
+      'desc' => "{GreenstoneSQLPlug.db_client_user}",
+      'type' => "string",	
+      'deft' => "root",
+      'reqd' => "yes"},
+    { 'name' => "db_client_pwd", 
+      'desc' => "{GreenstoneSQLPlug.db_client_pwd}",
+      'type' => "string",
+      'deft' => "",
+      'reqd' => "no"}, # pwd not required: can create mysql accounts without pwd
+    { 'name' => "db_host", 
+      'desc' => "{GreenstoneSQLPlug.db_host}",
+      'type' => "string",
+      'deft' => "127.0.0.1", # localhost doesn't work for us, but 127.0.0.1 works. See gsmysql.pm
+      'reqd' => "yes"},
+    { 'name' => "db_port", 
+      'desc' => "{GreenstoneSQLPlug.db_port}",
+      'type' => "string", # NOTE: make this int? No default for port, since it's not a required connection param
+      'reqd' => "no"}
+    ];
+
+my $options = { 'name'     => "GreenstoneSQLPlugout",
+		'desc'     => "{GreenstoneSQLPlugout.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+##### This entire class is called only during import.pl #####
+
+##### Overridden methods #####
+
+sub new {
+    my ($class) = shift (@_);
+    my ($plugoutlist, $inputargs,$hashArgOptLists) = @_;
+    push(@$plugoutlist, $class);
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    my $self = new GreenstoneXMLPlugout($plugoutlist,$inputargs,$hashArgOptLists);
+    
+    if ($self->{'info_only'}) {	
+        # don't worry about any options etc
+        return bless $self, $class;
+    }
+    
+    return bless $self, $class;
+}
+
+# connect here and ensure all tables and databases exist
+sub begin {
+
+    my $self= shift (@_);
+    
+    my $db_params = {
+	'collection_name' => $ENV{'GSDLCOLLECTION'},
+	'verbosity' => $self->{'verbosity'} || 0
+	
+    };
+
+    my $gs_sql = new gsmysql($db_params);
+
+    # if autocommit is set, there's no rollback support
+    my $autocommit = ($self->{'rollback_on_cancel'} eq "false") ? 1 : 0;
+    
+    # try connecting to the mysql db, die if that fails
+    # So don't bother preparing GreenstoneXMLPlugout by calling superclass' begin() yet
+    if(!$gs_sql->connect_to_db({
+	'db_driver' => $self->{'db_driver'},
+	'db_client_user' => $self->{'db_client_user'},
+	'db_client_pwd' => $self->{'db_client_pwd'},
+	'db_host' => $self->{'db_host'},
+	'db_port' => $self->{'db_port'}, # undef by default, can leave as is
+	'autocommit' => $autocommit
+			       })
+	)
+    {
+	# This is fatal for the plugout, let's terminate here
+	# PrintError would already have displayed the warning message on connection fail	
+	die("Could not connect to db. Can't proceed.\n");
+    }
+
+    my $db_name = $self->{'site'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
+    my $proc_mode = $self->{'process_mode'};
+    
+
+    my $success = $gs_sql->use_db($db_name);
+    
+    if($success && $proc_mode ne "text_only") {
+	$success = $gs_sql->ensure_meta_table_exists();
+    }    
+    if($success && $proc_mode ne "meta_only") {
+	$success = $gs_sql->ensure_fulltxt_table_exists();
+    }
+    
+    if(!$success) {	
+	# This is fatal for the plugout, let's terminate here after disconnecting again
+	# PrintError would already have displayed the warning message on load fail
+	# And on die() perl will call gsmysql destroy which will ensure a disconnect() from db
+	die("Could not use db $db_name and/or prepare its tables. Can't proceed.\n");
+    }
+
+    # store the DBI wrapper instance
+    $self->{'gs_sql'} = $gs_sql;
+    
+    
+    # If setting up connection to sql db failed, we'd have terminated and wouldn't come up to here
+    # and wouldn't have bothered preparing GreenstoneXMLPlugout by calling superclass' begin().
+    # Finally, can call begin on super - important as doc.xml is opened as a group etc
+    
+    $self->SUPER::begin(@_);
+}
+
+# disconnect from database here, see inexport.pm
+sub end
+{
+    my $self = shift(@_);
+
+    # do the superclass stuff first, as any sql db failures should not prevent superclass cleanup
+    $self->SUPER::end(@_);    
+
+    # Important to call finished():
+    # it will disconnect from db if this is the last gsmysql instance,
+    # and it will commit to db before disconnecting if rollbback_on_cancel turned on
+    $self->{'gs_sql'}->finished();
+    delete $self->{'gs_sql'}; # key gs_sql no longer exists, not just the value being undef
+}
+
+# Produce files called docsql.xml instead of doc.xml
+sub get_doc_xml_filename {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    
+    return "docsql.xml";
+}
+
+# overriding to store doc OID as attribute of top level element: <Archive docoid="oid">
+sub output_xml_header {
+    my $self = shift (@_);
+    my ($outhandle, $doc_oid) = @_;
+
+    print $outhandle '<?xml version="1.0" encoding="utf-8" standalone="no"?>' . "\n";
+    print $outhandle "<!DOCTYPE Archive SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
+    print $outhandle "<Archive docoid=\"$doc_oid\">\n";
+}
+ 
+# saveas() only generates the content in archives dir and in the SQL database
+sub saveas {
+    my $self = shift (@_);
+    my ($doc_obj, $doc_dir) = @_;
+
+    my $proc_mode = $self->{'process_mode'};
+    
+    # 1. pre save out and saving debug handle
+
+    # must call superclass (pre/post) saveas methods, as they handle assoc_files too
+    my ($docxml_outhandler, $output_file) = $self->SUPER::pre_saveas(@_);
+
+    $self->{'debug_outhandle'} = $docxml_outhandler if ($self->{'debug'}); # STDOUT if debug
+
+    # 2. overriding saving behaviour to do what the superclass does (writing out doc.xml files,
+    # under new name of docsql.xml, with breadcrumbs pointing to sql db) PLUS saving to sql db    
+
+    # NOTE: if proc_mode == all, then "breadcrumbs" (statements pointing viewer to the sql db
+    # for contents) go into both meta and txt elements of doc.xml (docsql.xml specifically): 
+	
+    # write the INVERSE into doc.xml as to what is written to the SQL db    
+    my $docxml_output_options = { 'output' => docprint::OUTPUT_NONE };
+    if($proc_mode eq "meta_only" ) { # since only meta to go into MySQL db, text will go into docxml
+	$docxml_output_options->{'output'} = docprint::OUTPUT_TEXT_ONLY;
+    } elsif($proc_mode eq "text_only" ) { # since only full text to go into MySQL db, meta will go into docxml
+	$docxml_output_options->{'output'} = docprint::OUTPUT_META_ONLY;
+    }
+    
+    # now we've prepared to write out whatever is meant to go into docxml
+    # and can do actual the steps superclass GreenstoneXMLPlugout carries out to write out docxml
+    # So: write out the doc xml file, "docsql.xml", for the current document
+    my $section_text = &docprint::get_section_xml($doc_obj, $docxml_output_options);
+    print $docxml_outhandler $section_text;    
+    
+    
+    # We also write out whatever needs to go into the MySQL database
+    $self->write_meta_and_text($doc_obj);
+
+    
+    # 3. post save out 
+    $self->SUPER::post_saveas($doc_obj, $doc_dir, $docxml_outhandler, $output_file);
+    
+    
+    # database connection is closed once, in end() method
+    # We're not opening and closing over and over for each doc during a single build
+}
+
+##### New methods, not inherited  #####
+
+# write meta and/or text PER DOC out to DB
+sub write_meta_and_text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $doc_oid = $doc_obj->get_OID(); # this method processes a single doc at a time, so it uses the same OID throughout
+    my $root_section = $doc_obj->get_top_section();
+
+    $self->recursive_write_meta_and_text($doc_obj, $doc_oid, $root_section);
+}
+
+sub recursive_write_meta_and_text {
+    my $self = shift (@_);
+    my ($doc_obj, $doc_oid, $section) = @_;    
+
+    # If section=ROOT, write "root" as section name into table
+    # doc->get_top_section() is the name of the doc root section, which is ""
+    my $section_name = ($section eq "") ? "root" : $section;
+    
+    my $section_ptr = $doc_obj->_lookup_section ($section);
+    return "" unless defined $section_ptr;
+
+    my $debug_out = $self->{'debug_outhandle'};
+
+    my $gs_sql = $self->{'gs_sql'};
+    my $proc_mode = $self->{'process_mode'};
+    if($proc_mode eq "all" || $proc_mode eq "meta_only" ) {
+	
+	##binmode(STDERR, ":utf8"); # shouldn't be necessary as we call &unicode::utf8decomp() to avoid wide-character warnings by printing wide chars as unicode codepoints
+	#print STDERR "###### dc.Title: ".&unicode::utf8decomp($doc_obj->get_metadata_element($doc_obj->get_top_section(), "dc.Title"))."\n";
+	
+	foreach my $data (@{$section_ptr->{'metadata'}}) {
+	    my $meta_name = $data->[0];
+
+	    # Treat db like a text file instead of an html/xml file: don't need to escape text
+	    # going into it, unlike with doc(sql).xml
+	    my $meta_value = $data->[1];
+		
+	    # Write out the current section's meta to collection db's METADATA table	    
+	    
+	    # For each set of values to write to meta table, this next method call will
+	    # efficiently execute an insert SQL statement (using a prepared insert statement), 
+	    # filling in the values
+	    # OR if debugging, then it will print the SQL insert statement but not execute it
+	    # (a behaviour following what the GS XML Plugout superclass does on debug)
+	    
+	    $gs_sql->insert_row_into_metadata_table($doc_oid, $section_name, $meta_name, $meta_value, $self->{'debug'});
+	}
+    }
+    
+    
+    if($proc_mode eq "all" || $proc_mode eq "text_only" ) {
+
+	# See above, no need to html-escape for db
+	my $section_text = $section_ptr->{'text'};
+	
+	# fulltxt column can be SQL NULL. undef value for $section_text gets written out as NULL:
+	# https://stackoverflow.com/questions/12708633/which-one-represents-null-undef-or-empty-string
+	# The following will do the SQL insertion
+	# or if debug, the following will print the SQL insert stmt without executing it
+	$gs_sql->insert_row_into_fulltxt_table($doc_oid, $section_name, \$section_text, $self->{'debug'});
+    
+    }
+    
+    # output all subsections: RECURSIVE CALL
+    foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
+	$self->recursive_write_meta_and_text($doc_obj, $doc_oid, "$section.$subsection");
+    }
+}
+
+
+1;
