Index: main/trunk/greenstone2/perllib/gssql.pm
===================================================================
--- main/trunk/greenstone2/perllib/gssql.pm	(revision 32562)
+++ main/trunk/greenstone2/perllib/gssql.pm	(revision 32563)
@@ -44,5 +44,4 @@
 
 # Parameterise (one or more methods may use them):
-# - build_mode (like removeold)
 # - db_name (which is the GS3 sitename)
 
@@ -134,5 +133,5 @@
 	print STDERR "Away to make connection to $db_driver database with:\n";
 	print STDERR " - hostname $db_host; username: $db_user";
-	print STDERR "; and the password provided" if $db_pwd;
+	print STDERR "; and the provided password" if $db_pwd;
 	print STDERR "\nAssuming the mysql server has been started with: --character_set_server=utf8mb4\n" if $db_driver eq "mysql";
     }
@@ -188,10 +187,12 @@
 }
 
-# will attempt to load the specified db and the <coll>_metadata and <coll>_fulltxt for this
-# collection, or create any of these (db, tables) that don't yet exist. At the end
-# it will have loaded the requested database (in MySQL: "use <db>;")
-sub load_db_and_tables {
-    my $self= shift (@_);
-    my ($db_name, $build_mode) = @_;
+# Load the designated database, i.e. 'use <dbname>;'.
+# If the database doesn't yet exist, creates it and loads it.
+# (Don't create the collection's tables yet, though)
+# At the end it will have loaded the requested database (in MySQL: "use <db>;") on success.
+# As usual, returns success or failure value that can be evaluated in a boolean context.
+sub use_db {
+    my $self= shift (@_);
+    my ($db_name) = @_;
     my $dbh = $self->{'db_handle'};
     $db_name = $self->sanitize_name($db_name);
@@ -215,9 +216,5 @@
 	# once more attempt to use db, now that it exists
 	$dbh->do("use $db_name") || return 0;
-	#$dbh->do("use localsite") or die "Error (code" . $dbh->err ."): " . $dbh->errstr . "\n";
-
-	# attempt to create tables in current db
-	$self->create_metadata_table() || return 0;
-	$self->create_fulltext_table() || return 0;	
+	#$dbh->do("use $db_name") or die "Error (code" . $dbh->err ."): " . $dbh->errstr . "\n";
 
 	$success = 1;
@@ -227,26 +224,4 @@
 
 	print STDERR "@@@ DATABASE $db_name EXISTED\n" if($self->{'verbosity'} > 2);
-	
-
-	# build_mode can be removeold or incremental. We only do something special on removeold:
-	# deleting the existing tables for this collection and recreating empty ones
-	if($build_mode eq "removeold") {
-	    print STDERR "   Building with removeold option set, so deleting current collection's tables if they exist\n" if($self->{'verbosity'});
-	    $self->delete_collection_tables();
-	}
-
-	# use existing tables if any
-	# attempt to create tables in current db	
-	if($build_mode eq "removeold" || !$self->table_exists($self->get_metadata_table_name())) {
-	    $self->create_metadata_table() || return 0;
-	} else {
-	    print STDERR "@@@ Meta table exists\n" if($self->{'verbosity'} > 2);
-	}
-	if($build_mode eq "removeold" || !$self->table_exists($self->get_fulltext_table_name())) {
-	    $self->create_fulltext_table() || return 0;
-	} else {
-	    print STDERR "@@@ Fulltxt table exists\n" if($self->{'verbosity'} > 2);
-	}
-	
     }
     
@@ -254,23 +229,32 @@
 }
 
-# GreenstoneSQLPlugin calls this method to load an existing db.
-# This will terminate if the db does not exist. Unlike load_db_and_tables() above, used by
-# GreenstoneSQLPlugout, this method will not attempt to create the requested db (nor its tables)
-# TODO: GS SQLPlugin is called before GS SQLPlugout and attempts to use_db() - called in plugin's
-# init() method. This will fail if the db does not exist. Ideally want the gssqlplugin only called
-# during buildcol.pl
-sub use_db {
-    my $self= shift (@_);
-    my ($db_name) = @_;
-    my $dbh = $self->{'db_handle'};
-    $db_name = $self->sanitize_name($db_name);
-
-    
-    print STDERR "Loading database $db_name\n" if($self->{'verbosity'} > 1);
-    
-    # perl DBI switch database: https://www.perlmonks.org/?node_id=995434
-    # do() returns undef on error.
-    # connection succeeded, try to load our database. If that didn't work, attempt to create db
-    return $dbh->do("use $db_name") || warn();
+# We should already have done "use <database>;" if this gets called.
+# Just load this collection's metatable
+sub ensure_meta_table_exists {
+    my $self = shift (@_);
+    
+    my $tablename = $self->get_metadata_table_name();
+    if(!$self->table_exists($tablename)) {
+	#print STDERR "   Creating metadata table $tablename\n" if($self->{'verbosity'} > 1);
+	$self->create_metadata_table() || return 0;
+    } else {
+	print STDERR "@@@ Meta table exists\n" if($self->{'verbosity'} > 2);
+    }
+    return 1;
+}
+
+# We should already have done "use <database>;" if this gets called.
+# Just load this collection's metatable
+sub ensure_fulltxt_table_exists {
+    my $self = shift (@_);
+    
+    my $tablename = $self->get_fulltext_table_name();    
+    if(!$self->table_exists($tablename)) {
+	#print STDERR "   Creating fulltxt table $tablename\n" if($self->{'verbosity'} > 1);
+	$self->create_fulltext_table() || return 0;
+    } else {
+	print STDERR "@@@ Fulltxt table exists\n" if($self->{'verbosity'} > 2);
+    }
+    return 1;
 }
 
Index: main/trunk/greenstone2/perllib/inexport.pm
===================================================================
--- main/trunk/greenstone2/perllib/inexport.pm	(revision 32562)
+++ main/trunk/greenstone2/perllib/inexport.pm	(revision 32563)
@@ -690,5 +690,4 @@
     }
     my $processor = &plugout::load_plugout($plugout);
-    $processor->set_incremental_options($removeold, $keepold, $incremental, $incremental_mode);
     $processor->setoutputdir ($archivedir);
     $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
Index: main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm	(revision 32562)
+++ main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm	(revision 32563)
@@ -40,6 +40,4 @@
 # TODO:
 # - Run TODOs here, in Plugout and in gssql.pm by Dr Bainbridge.
-# - "Courier" demo documents in lucene-sql collection: character (degree symbol) not preserved in title. Is this because we encode in utf8 when putting into db and reading back in?
-# Test doc with meta and text like macron in Maori text.
 # - Have not yet tested writing out just meta or just fulltxt to sql db and reading just that 
 # back in from the sql db while the remainder is to be read back in from the docsql .xml files.
@@ -47,12 +45,4 @@
 # TODO: deal with incremental vs removeold. If docs removed from import folder, then import step
 # won't delete it from archives but buildcol step will. Need to implement this with this database plugin or wherever the actual flow is
-
-# TODO: Add public instructions on using this plugin and its plugout: start with installing mysql binary, changing pwd, running the server (and the client against it for checking, basic cmds like create and drop). Then discuss db name, table names (per coll), db cols and col types, and how the plugout and plugin work.
-# Discuss the plugin/plugout parameters.
-
-# + NOTTODO: when db is not running GLI is paralyzed -> can we set timeout on DBI connection attempt?
-# NOT A PROBLEM: Tested to find DBI connection attempt fails immediately when MySQL server not 
-# running. The GLI "paralyzing" incident last time was not because of the gs sql connection code,
-# but because my computer was freezing on-and-off.
 
 # TODO Q: is "reindex" = del from db + add to db?
@@ -63,9 +53,9 @@
 # be reimplemented in GSSQLPlugin to support the adding portion of reindexing.
 
-
-# TODO Q: During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
-# effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
-
-
+# TODO: Add public instructions on using this plugin and its plugout: start with installing mysql binary, changing pwd, running the server (and the client against it for checking: basic cmds like create and drop). Then discuss db name, table names (per coll), db cols and col types, and how the plugout and plugin work.
+# Discuss the plugin/plugout parameters.
+
+
+# DONE:
 # + TODO: Incremental delete can't work until GSSQLPlugout has implemented build_mode = incremental
 # (instead of tossing away db on every build)
@@ -79,4 +69,13 @@
 # (sections) which in this case had made it easy to reconstruct the doc_obj in memory in the correct order.
 # YES: Otherwise for later db types (drivers), can set order by primary key column and then order by did column
+# + NOTTODO: when db is not running GLI is paralyzed -> can we set timeout on DBI connection attempt?
+#   NOT A PROBLEM: Tested to find DBI connection attempt fails immediately when MySQL server not 
+# running. The GLI "paralyzing" incident last time was not because of the gs sql connection code,
+# but because my computer was freezing on-and-off.
+# + "Courier" demo documents in lucene-sql collection: character (degree symbol) not preserved in title. Is this because we encode in utf8 when putting into db and reading back in?
+# Test doc with meta and text like macron in Maori text.
+# + TODO Q: During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
+# effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
+#   This got fixed when GSSQLPlugin stopped connecting on init().
 
 
@@ -177,78 +176,61 @@
 
     # do anything else that needs to be done here when not pluginfo
-    #$self->{'delete_docids'} = (); # list of doc oids to delete during deinit()
     
     return $self;
 }
 
-sub xml_start_tag {
-    my $self = shift(@_);
-    my ($expat, $element) = @_;
-
-    my $outhandle = $self->{'outhandle'};
-    
-    $self->{'element'} = $element;
-    if ($element eq "Archive") { # docsql.xml files contain a OID attribute on Archive element
-	# the element's attributes are in %_ as per ReadXMLFile::xml_start_tag() (while $_
-	# contains the tag)
-
-	# Don't access %_{'docoid'} directly: keep getting a warning message to 
-	# use $_{'docoid'} for scalar contexts, but %_ is the element's attr hashmap 
-	# whereas $_ has the tag info. So we don't want to do $_{'docoid'}.
-	my %attr_hash = %_; # right way, see OAIPlugin.pm
-	$self->{'doc_oid'} = $attr_hash{'docoid'};
-	print STDERR "XXXXXXXXXXXXXX in SQLPlugin::xml_start_tag()\n";
-	print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
-	    if $self->{'verbosity'} > 2;
-
-    }
-    else { # let superclass GreenstoneXMLPlugin continue to process <Section> and <Metadata> elements
-	$self->SUPER::xml_start_tag(@_);
-    }
-}
-
-# TODO Q: Why are there 4 passes when we're only indexing at doc and section level (2 passes)? What's the dummy pass, why is there a pass for infodb?
-
-# At the end of superclass GreenstoneXMLPlugin.pm's close_document() method,
-# the doc_obj in memory is processed (indexed) and then made undef.
-# So we have to work with doc_obj before superclass close_document() is finished.
-sub close_document {
-    my $self = shift(@_);
-
-    print STDERR "XXXXXXXXX in SQLPlugin::close_doc()\n";
-    
-    my $gs_sql = $self->get_gssql_instance(); 
-    
-    my $outhandle = $self->{'outhandle'};
-    my $doc_obj = $self->{'doc_obj'};
-
-    my $build_proc_mode = $self->{'processor'}->get_mode(); # can be "text" as per basebuildproc or infodb
-    my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
+# This is called once if removeold is set with import.pl. Most plugins will do
+# nothing but if a plugin does any stuff outside of creating doc obj, then 
+# it may need to clear something.
+# In the case of GreenstoneSQL plugs: this is the first time we have a chance
+# to purge the tables of the current collection from the current site's database
+sub remove_all {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+
+    print STDERR "   Building with removeold option set, so deleting current collection's tables if they exist\n" if($self->{'verbosity'});
+    
+    # if we're in here, we'd already have run 'use database <site_name>;' during sub init()
+    # so we can go ahead and delete the collection's tables
+    my $gs_sql = $self->{'gs_sql'};
+    $gs_sql->delete_collection_tables(); # will delete them if they exist
+
+    # and recreate tables? No. Tables' existence is ensured in GreenstoneSQLPlugout::begin()
     my $proc_mode = $self->{'process_mode'};
-    
-    print $outhandle "++++ OID of document (meta|text) to be del or read in from DB: ".$self->{'doc_oid'}."\n"
-	if $self->{'verbosity'} > 2;
-    
-    # For now, we have access to doc_obj (until just before super::close_document() terminates)
-    
-    # no need to call $self->{'doc_obj'}->set_OID($oid);
-    # because either the OID is stored in the SQL db as meta 'Identifier' alongside other metadata
-    # or it's stored in the doc.xml as metadata 'Identifier' alongside other metadata
-    # Either way, Identifier meta will be read into the docobj automatically with other meta.
-    
-    if ($self->{'verbosity'} > 2) {
-	print STDERR "+++++++++++ buildproc_mode: $build_proc_mode\n";
-	print STDERR "+++++++++++ SQLPlug proc_mode: $proc_mode\n";
-    }
-
-    # TODO: where does reindexing take place, GreenstoneSQL -Plugout or -Plugin?
-    #if($build_proc_mode =~ m/(delete|reindex)$/) { # doc denoted by current OID has been marked for deletion or reindexing (=delete + add)
-    if($build_proc_mode =~ m/(delete)$/) { # doc denoted by current OID has been marked for deletion or reindexing (=delete + add)
-	
-	      # build_proc_mode could be "(infodb|text)(delete|reindex)"
-	      # "...delete" or "...reindex" as per ArchivesInfPlugin
-	
-	print STDERR "@@@@ DELETING DOC FROM SQL DB\n";
-	
+    if($proc_mode ne "text_only") {
+	$gs_sql->ensure_meta_table_exists();
+    }
+    if($proc_mode ne "meta_only") {
+	$gs_sql->ensure_fulltxt_table_exists();
+    }
+}
+
+# This is called per document for docs that have been deleted from the 
+# collection. Most plugins will do nothing
+# but if a plugin does any stuff outside of creating doc obj, then it may need
+# to clear something.
+# remove the doc(s) denoted by oids from GS SQL db
+# This takes care of incremental deletes (docs marked D by ArchivesInfPlugin when building
+# incrementally) as well as cases of "Non-icremental Delete", see ArchivesInfPlugin.pm
+sub remove_one {
+    my $self = shift (@_);
+    
+    my ($file, $oids, $archivedir) = @_;
+
+    print STDERR "@@@ IN SQLPLUG::REMOVE_ONE: $file\n";
+    
+    #return undef unless $self->can_process_this_file($file); # NO, DON'T DO THIS: 
+           # WE DON'T CARE IF IT'S AN IMAGE FILE THAT WAS DELETED.
+           # WE CARE ABOUT REMOVING THE DOCOID OF THAT IMAGE FILE FROM THE DB
+
+    print STDERR "*****************************\nAsked to remove_one oid\n***********************\n";
+    
+    my $gs_sql = $self->{'gs_sql'} || return 0; # couldn't make the connection or no db etc
+    if(scalar @$oids > 1) {
+	print STDERR "TODO: We now have reason to optimise GreenstoneSQLPlugin::remove_one() by using prepare and execute.\n";
+    }
+    
+    my $proc_mode = $self->{'process_mode'};
+    foreach my $oid (@$oids) {	
 	if($proc_mode eq "all" || $proc_mode eq "meta_only") {
 	    print STDERR "@@@@@@@@ Deleting $oid from meta table\n" if $self->{'verbosity'} > 2;
@@ -259,70 +241,118 @@
 	    $gs_sql->delete_recs_from_texttable_with_docid($oid);
 	}
-	
-	# If we're reindexing the current doc, we will we want to continue: which
-	# will add this doc ID back into the db with the new meta/full txt values
-	# But if we're deleting, then we're done processing the document, so set doc_oid to undef
-	# to prevent adding it back into db
-	#undef $self->{'doc_oid'} if($build_proc_mode =~ m/delete$/);	
-	
-    } # done deleting doc from SQL db    
-    
-    else {#if($self->{'doc_oid'}) { # if loading doc from SQL db
-	print STDERR "@@@@ LOADING DOC FROM SQL DB\n";	
-	
-	if($proc_mode eq "all" || $proc_mode eq "meta_only") {
-	    # read in meta for the collection (i.e. select * from <col>_metadata table
+    }
+    return 1;
+}
+
+
+sub xml_start_tag {
+    my $self = shift(@_);
+    my ($expat, $element) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    
+    $self->{'element'} = $element;
+    if ($element eq "Archive") { # docsql.xml files contain a OID attribute on Archive element
+	# the element's attributes are in %_ as per ReadXMLFile::xml_start_tag() (while $_
+	# contains the tag)
+
+	# Don't access %_{'docoid'} directly: keep getting a warning message to 
+	# use $_{'docoid'} for scalar contexts, but %_ is the element's attr hashmap 
+	# whereas $_ has the tag info. So we don't want to do $_{'docoid'}.
+	my %attr_hash = %_; # right way, see OAIPlugin.pm
+	$self->{'doc_oid'} = $attr_hash{'docoid'};
+	##print STDERR "XXXXXXXXXXXXXX in SQLPlugin::xml_start_tag()\n";
+	print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
+	    if $self->{'verbosity'} > 2;
+
+    }
+    else { # let superclass GreenstoneXMLPlugin continue to process <Section> and <Metadata> elements
+	$self->SUPER::xml_start_tag(@_);
+    }
+}
+
+# TODO Q: Why are there 4 passes when we're only indexing at doc and section level (2 passes)? What's the dummy pass, why is there a pass for infodb?
+
+# We should only ever get here during the buildcol.pl phase
+# At the end of superclass GreenstoneXMLPlugin.pm's close_document() method,
+# the doc_obj in memory is processed (indexed) and then made undef.
+# So we have to work with doc_obj before superclass close_document() is finished.
+sub close_document {
+    my $self = shift(@_);
+
+    ##print STDERR "XXXXXXXXX in SQLPlugin::close_doc()\n";
+    
+    my $gs_sql = $self->{'gs_sql'};
+    
+    my $outhandle = $self->{'outhandle'};
+    my $doc_obj = $self->{'doc_obj'};
+
+    my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
+    my $proc_mode = $self->{'process_mode'};
+    
+    # For now, we have access to doc_obj (until just before super::close_document() terminates)
+    
+    # no need to call $self->{'doc_obj'}->set_OID($oid);
+    # because either the OID is stored in the SQL db as meta 'Identifier' alongside other metadata
+    # or it's stored in the doc.xml as metadata 'Identifier' alongside other metadata
+    # Either way, Identifier meta will be read into the docobj automatically with other meta.
+
+    print STDERR "   GreenstoneSQLPlugin processing doc $oid (reading into docobj from SQL db)\n"
+	if $self->{'verbosity'} > 0;
+    
+    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+	# read in meta for the collection (i.e. select * from <col>_metadata table
+	
+	my $sth = $gs_sql->select_from_metatable_matching_docid($oid);
+	print $outhandle "### SQL select stmt: ".$sth->{'Statement'}."\n"
+	    if $self->{'verbosity'} > 2;
+	
+	print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
+	# https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
+	while( my @row = $sth->fetchrow_array() ) {	    
+	    #print $outhandle "row: @row\n";
+	    my ($primary_key, $did, $sid, $metaname, $metaval) = @row;
 	    
-	    my $sth = $gs_sql->select_from_metatable_matching_docid($oid);
-	    print $outhandle "### SQL select stmt: ".$sth->{'Statement'}."\n"
+	    # get rid of the artificial "root" introduced in section id when saving to sql db
+	    $sid =~ s@^root@@;
+	    $sid = $doc_obj->get_top_section() unless $sid;
+	    print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
 		if $self->{'verbosity'} > 2;
 	    
-	    print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
-	    # https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
-	    while( my @row = $sth->fetchrow_array() ) {	    
-		#print $outhandle "row: @row\n";
-		my ($primary_key, $did, $sid, $metaname, $metaval) = @row;
+	    # TODO:  we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_meta directly:
+	    $doc_obj->add_utf8_metadata($sid, $metaname, &docprint::unescape_text($metaval));
+	}
+	print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
+	    if $self->{'verbosity'} > 2;
+    }
+    
+    if($proc_mode eq "all" || $proc_mode eq "text_only") {
+	# read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
+	
+	my $fulltxt_table = $gs_sql->get_fulltext_table_name();
+	
+	
+	my $sth = $gs_sql->select_from_texttable_matching_docid($oid);
+	print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 2;
+	
+	print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
+	    if $self->{'verbosity'} > 2;
+	while( my ($primary_key, $did, $sid, $text) = $sth->fetchrow_array() ) {	   
 	    
-		# get rid of the artificial "root" introduced in section id when saving to sql db
-		$sid =~ s@^root@@;
-		$sid = $doc_obj->get_top_section() unless $sid;
-		print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
-		    if $self->{'verbosity'} > 2;
-		
-		# TODO:  we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_meta directly:
-		$doc_obj->add_utf8_metadata($sid, $metaname, &docprint::unescape_text($metaval));
-	    }
-	    print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
+	    # get rid of the artificial "root" introduced in section id when saving to sql db
+	    #$sid =~ s@^root@@;
+	    $sid = $doc_obj->get_top_section() if ($sid eq "root");
+	    print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
 		if $self->{'verbosity'} > 2;
-	}
-	
-	if($proc_mode eq "all" || $proc_mode eq "text_only") {
-	    # read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
 	    
-	    my $fulltxt_table = $gs_sql->get_fulltext_table_name();
-	    
-	
-	    my $sth = $gs_sql->select_from_texttable_matching_docid($oid);
-	    print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 2;
-	    
-	    print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
-		if $self->{'verbosity'} > 2;
-	    while( my ($primary_key, $did, $sid, $text) = $sth->fetchrow_array() ) {	   
-		
-		# get rid of the artificial "root" introduced in section id when saving to sql db
-		#$sid =~ s@^root@@;
-		$sid = $doc_obj->get_top_section() if ($sid eq "root");
-		print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
-		    if $self->{'verbosity'} > 2;
-		
-		# TODO - pass by ref?
-		# TODO: we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_text directly:
-		$doc_obj->add_utf8_text($sid, &docprint::unescape_text($text));
-	    }	
-	    print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
-		if $self->{'verbosity'} > 2;
-	}
-	
-    } # done reading into docobj from SQL db
+	    # TODO - pass by ref?
+	    # TODO: we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_text directly:
+	    $doc_obj->add_utf8_text($sid, &docprint::unescape_text($text));
+	}	
+	print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
+	    if $self->{'verbosity'} > 2;
+    }
+    
+    # done reading into docobj from SQL db
     
     # don't forget to clean up on close() in superclass
@@ -332,33 +362,25 @@
 
 
-# We want SQLPlugin to connect to db only during buildcol.pl phase, not during import.pl
-# This works out okay, as close_document() (called by read()) is only invoked during buildcol.pl
-#
-# Further, we want a single db connection for the GS SQL Plugin to be used for 
-# the multiple plugin passes: for "dummy" pass, and for doc level and for section level indexing
-# By calling the lazy loading get_sql_instance() from close_document(),
-# we connect to the SQL database once per GSSQLPlugin and only during the buildcol phase.
-#
-# get_gssql_instance() is a lazy loading method that returns singleton db connection for a GreenstoneSQLPlugin object. ("Code pattern" get instance vs singleton.)
-# One instance of db connection that can be used for all the many doc_objects processed by this plugin
-#
-# Except in methods get_gssql_instance() and deinit(), don't access self->{'_gs_sql'} directly.
-# Instead, call method get_gssql_instance() and store return value in a local variable, my $gs_sql
-#
-sub get_gssql_instance
-{    
-    my $self = shift(@_);
-
-    # if we failed to successfully connect once before, don't bother attempting to connect again
-    #return undef if(defined $self->{'failed'}); # plugin/process would have terminated with die()
-                                  # if we couldn't succeed connecting on any connection attempt
-    
-    return $self->{'_gs_sql'} if($self->{'_gs_sql'});
-
-    # assume we'll fail to connect
-    $self->{'failed'} = 1;
-
-    print STDERR "@@@@@@@@@@ LAZY CONNECT CALLED\n";
-    
+# TODO: only want to work with sql db if buildcol.pl. Unfortunately, also runs on import.pl.
+# During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
+# effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
+
+# GS SQL Plugin::init() (and deinit()) is called by import.pl and also by buildcol.pl
+# This means it connects and deconnects during import.pl as well. This is okay
+# as removeold, which should drop the collection tables, happens during the import phase
+# and therefore also requires a db connection.
+# TODO: Eventually can try moving get_gssql_instance into gssql.pm? That way both GS SQL Plugin
+# and Plugout would be using one connection during import.pl phase when both plugs exist.
+
+# Call init() not begin() because there can be multiple plugin passes and begin() called for
+# each pass (one for doc level and another for section level indexing), whereas init() should
+# be called before any and all passes.
+# This way, we can connect to the SQL database once per buildcol run.
+sub init {
+    my ($self) = shift (@_);
+    ##print STDERR "@@@@@@@@@@ INIT CALLED\n";
+    
+    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
+
     ####################
 #    print "@@@ SITE NAME: ". $self->{'site_name'} . "\n" if defined $self->{'site_name'};
@@ -393,7 +415,7 @@
     
     my $db_name = $self->{'site_name'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
-    #my $build_mode = $self->{'build_mode'} || "removeold";
-
-    # the db and its tables should exist. Attempt to use the db:
+
+    # Attempt to use the db, create it if it doesn't exist (but don't create the tables yet)
+    # Bail if we can't use the database
     if(!$gs_sql->use_db($db_name)) {
 	
@@ -404,12 +426,10 @@
 	die("Could not use db $db_name. Can't proceed.\n");
     }
-
-    #undef $self->{'failed'};
+    
     
     # store db handle now that we're connected
-    $self->{'_gs_sql'} = $gs_sql;
-    return $gs_sql;
-    
-}
+    $self->{'gs_sql'} = $gs_sql;    
+}
+
 
 # This method also runs on import.pl if gs_sql has a value. But we just want to run it on buildcol
@@ -421,13 +441,14 @@
     my ($self) = shift (@_);
     
-    print STDERR "@@@@@@@@@@ GreenstoneSQLPlugin::DEINIT CALLED\n";
-    
-    if($self->{'_gs_sql'}) { # only want to work with sql db if buildcol.pl, gs_sql won't have
+    ##print STDERR "@@@@@@@@@@ GreenstoneSQLPlugin::DEINIT CALLED\n";
+    
+    if($self->{'gs_sql'}) { # only want to work with sql db if buildcol.pl, gs_sql won't have
 	# a value except during buildcol, so when processor =~ m/buildproc$/.
-	$self->{'_gs_sql'}->disconnect_from_db()
+	$self->{'gs_sql'}->disconnect_from_db()
 	    || warn("Unable to disconnect from database " . $self->{'site_name'} . "\n");
 
-	# explicitly set to undef so all future use has to make the connection again
-	undef $self->{'_gs_sql'};
+	# explicitly delete gs_sql key (setting key to undef has a different meaning from deleting)
+	# so all future use has to make the connection again
+	delete $self->{'gs_sql'};
     }
 
Index: main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 32562)
+++ main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 32563)
@@ -347,14 +347,4 @@
 }
 
-# GreenstoneSQLPlugout needs to know whether we're doing removeold or not
-sub set_incremental_options {
-    my $self= shift (@_);
-    my ($removeold, $keepold, $incremental, $incremental_mode) = @_;
-    
-    $self->{'removeold'}        = $removeold;
-    $self->{'keepold'}          = $keepold;
-    $self->{'incremental'}      = $incremental;
-    $self->{'incremental_mode'} = $incremental_mode;
-}
 
 # OIDtype may be "hash" or "hash_on_full_filename" or "incremental" or "filename" or "dirname" or "full_filename" or "assigned"
Index: main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm	(revision 32562)
+++ main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm	(revision 32563)
@@ -53,5 +53,6 @@
 # It's fine: the die() stmts all take place before setting up the super class' begin
 
-# TODO Q: about build_mode: how to detect removeold
+# TODO Q: about build_mode: how to detect removeold. Now handled by 
+#   GreenstoneSQLPlugout::remove_all(), which is inherited from a base plugin.
 # TODO: deal with -removeold and everything? Or type out instructions for user
 
@@ -117,5 +118,5 @@
         return bless $self, $class;
     }
-    print STDERR "***** GreenstoneSQLPlugout process mode = \"", $self->{'process_mode'}, "\"\n";
+    #print STDERR "***** GreenstoneSQLPlugout process mode = \"", $self->{'process_mode'}, "\"\n";
     
     return bless $self, $class;
@@ -163,9 +164,19 @@
     
     my $db_name = $self->{'site_name'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
-    my $build_mode = ($self->{'removeold'}) ? "removeold" : "incremental";
-    print STDERR "@@@@@@@@@@@@ remove_old: $build_mode\n";
-    
-    if(!$gs_sql->load_db_and_tables($db_name, $build_mode)) {
-	
+    my $proc_mode = $self->{'process_mode'};
+    
+
+    my $success = $gs_sql->use_db($db_name);
+    
+    if($success && $proc_mode ne "text_only") {
+	##print STDERR "@@@@ Ensuring meta table exists\n";
+	$success = $gs_sql->ensure_meta_table_exists();
+    }    
+    if($success && $proc_mode ne "meta_only") {
+	##print STDERR "@@@@ Ensuring fulltxt table exists\n";
+	$success = $gs_sql->ensure_fulltxt_table_exists();
+    }
+    #if(!$gs_sql->load_db_and_tables($db_name, $proc_mode)) {
+    if(!$success) {	
 	# This is fatal for the plugout, let's terminate here after disconnecting again
 	# PrintError would already have displayed the warning message on load fail
@@ -177,5 +188,5 @@
     # prepare the shared/common HANDLES to SQL insert statements that contain placeholders
     # and which we will reuse repeatedly when actually executing the insert statements
-    my $proc_mode = $self->{'process_mode'};
+    
     if($proc_mode eq "all" || $proc_mode eq "meta_only" ) {
 	$self->{'metadata_prepared_insert_statement_handle'} = $gs_sql->prepare_insert_metadata_row_stmthandle();
@@ -188,6 +199,6 @@
     $self->{'gs_sql'} = $gs_sql;
     
-    print STDERR "#### Meta stmt: " . $self->{'metadata_prepared_insert_statement_handle'}->{'Statement'} . "\n";
-    print STDERR "#### Full stmt: " . $self->{'fulltxt_prepared_insert_statement_handle'}->{'Statement'} . "\n";
+    ##print STDERR "#### Meta stmt: " . $self->{'metadata_prepared_insert_statement_handle'}->{'Statement'} . "\n";
+    ##print STDERR "#### Full stmt: " . $self->{'fulltxt_prepared_insert_statement_handle'}->{'Statement'} . "\n";
     
     # if setting up to work with sql db failed, we'd have terminated and wouldn't come up to here:
