Index: /main/trunk/greenstone2/perllib/plugouts/MySQLPlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/MySQLPlugout.pm	(revision 32520)
+++ /main/trunk/greenstone2/perllib/plugouts/MySQLPlugout.pm	(revision 32521)
@@ -32,5 +32,4 @@
 use util;
 use FileUtils;
-#use BasePlugout;
 use GreenstoneXMLPlugout;
 use docprint;
@@ -38,4 +37,7 @@
 use IPC::Open2;
 use POSIX ":sys_wait_h"; # for waitpid, http://perldoc.perl.org/functions/waitpid.html
+
+# TODO: SIGTERM rollback and disconnect?
+
 
 # this plugout does not output xml to a file, but outputs rows into a mysql table
@@ -100,8 +102,9 @@
     $self->{'db_driver'} = "mysql";
     $self->{'site_name'} = "localsite";    
-    $self->{'client_user'} = "root";
-    $self->{'client_pwd'} = "6reenstone3";
+    $self->{'db_client_user'} = "root";
+    $self->{'db_client_pwd'} = "6reenstone3";
     #$self->{'db_host'} = "127.0.0.1";
     #$self->{'db_encoding'} = "utf8";
+    #TODO: proc_mode is also a saveas option
     
     ############ LOAD NECESSARY OPTIONS ###########
@@ -121,5 +124,5 @@
 
     # prepare the shared/common HANDLES to SQL insert statements that contain placeholders
-    # and which we will reuse repeatedly when executing the actual insert statements
+    # and which we will reuse repeatedly when actually executing the insert statements
     my $proc_mode = $self->{'process_mode'};
     if($proc_mode eq "all" || $proc_mode eq "meta_only" ) {
@@ -129,7 +132,12 @@
 	$self->{'fulltxt_prepared_insert_statement_handle'} = $self->prepare_insert_fulltxt_row_stmthandle();
     }
-    
-    # finally, call begin on super
-    $self->GreenstoneXMLPlugout::begin(@_);    
+
+    # if setting up to work with sql db failed, we'd have terminated and wouldn't come up to here:
+    # won't bother preparing GreenstoneXMLPlugout by calling superclass' begin()
+    # finally, can call begin on super - important as doc.xml is opened as a group etc
+
+    if($proc_mode ne "all") { # TODO Q
+	$self->GreenstoneXMLPlugout::begin(@_);
+    }
 }
 
@@ -139,12 +147,21 @@
     my $self = shift(@_);
 
-    # do the superclass stuff
-    $self->GreenstoneXMLPlugout::end(@_);
-    $self->disconnect_from_db() || warn("Unable to disconnect from database " . $self->{'site_name'} . "\n");
-    #my $success = $self->GreenstoneXMLPlugout::end(@_);
-    #$success = $self->disconnect_from_db() && $success;    
-    #return $success;
-}
-    
+    # do the superclass stuff first, as any sql db failures should not prevent superclass cleanup
+    
+    if($self->{'process_mode'} ne "all") {# TODO Q
+	$self->GreenstoneXMLPlugout::end(@_);
+    }
+    
+    $self->disconnect_from_db() || warn("Unable to disconnect from database " . $self->{'site_name'} . "\n"); # disconnect_from_db() will also issue a warning, but this may be clearer
+}
+ 
+sub close_group_output {
+    my $self = shift (@_);
+    
+    if($proc_mode ne "all") { # TODO Q
+	$self->GreenstoneXMLPlugout::close_group_output();
+    }
+}
+   
 # TODO: check arc-inf.db for whether each entry is to be deleted/indexed/reindexed/been indexed
 sub saveas {
@@ -152,35 +169,49 @@
     my ($doc_obj, $doc_dir) = @_;
 
-    # pre save out
-    my ($docxml_outhandler, $output_file) = $self->GreenstoneXMLPlugout::pre_saveas(@_);
-    
-
-    # saving customisation
-    $self->{'debug_outhandle'} = $docxml_outhandler if ($self->{'debug'}); # STDOUT if debug
-
-    # write the INVERSE into doc.xml as to what is written to the db
-    my $proc_mode = $self->{'process_mode'};
-    my $docxml_output_options = { 'output' => docprint::OUTPUT_NONE };
-    if($proc_mode eq "meta_only" ) { # since only meta to go into MySQL db, text will go into docxml
-	$docxml_output_options->{'output'} = docprint::OUTPUT_TEXT_ONLY;
-    } elsif($proc_mode eq "text_only" ) { # since only full text to go into MySQL db, meta will go into docxml
-	$docxml_output_options->{'output'} = docprint::OUTPUT_META_ONLY;
-    }
-
-    # now we've prepared to write out whatever is meant to go into docxml
-    # and can do actual the steps superclass GreenstoneXMLPlugout carries out to write out docxml
-    # So: write out the doc xml file for the current document
-    my $section_text = &docprint::get_section_xml($doc_obj, $docxml_output_options);
-    print $docxml_outhandler $section_text;    
-
+    # 1. pre save out and saving debug handle
+    if($proc_mode ne "all") { # TODO Q
+	my ($docxml_outhandler, $output_file) = $self->GreenstoneXMLPlugout::pre_saveas(@_);
+
+	$self->{'debug_outhandle'} = $docxml_outhandler if ($self->{'debug'}); # STDOUT if debug
+    }
+    elsif ($self->{'debug'}) { # if we didn't call superclass::pre_saveas, then set debug handle
+	$self->{'debug_outhandle'} = STDOUT; # superclass also debugs to STDOUT
+	# TODO: also set debugging in begin()? Then stmts creating db and tables also sent to debug out and not executed
+    }
+    
+    
+    # 2. overriding saving behaviour to do what the superclass does PLUS saving to sql db    
+
+    if($proc_mode ne "all" ) { # if all, then nothing to go into doc.xml
+	# write the INVERSE into doc.xml as to what is written to the db
+	my $proc_mode = $self->{'process_mode'};
+	
+	my $docxml_output_options = { 'output' => docprint::OUTPUT_NONE }; # TODO Q
+	
+	if($proc_mode eq "meta_only" ) { # since only meta to go into MySQL db, text will go into docxml
+	    $docxml_output_options->{'output'} = docprint::OUTPUT_TEXT_ONLY;
+	} elsif($proc_mode eq "text_only" ) { # since only full text to go into MySQL db, meta will go into docxml
+	    $docxml_output_options->{'output'} = docprint::OUTPUT_META_ONLY;
+	}
+
+	# now we've prepared to write out whatever is meant to go into docxml
+	# and can do actual the steps superclass GreenstoneXMLPlugout carries out to write out docxml
+	# So: write out the doc xml file for the current document
+	my $section_text = &docprint::get_section_xml($doc_obj, $docxml_output_options);
+	print $docxml_outhandler $section_text;    
+    }
+    
     # We also write out whatever needs to go into the MySQL database
     $self->write_meta_and_text($doc_obj);
 
     
-    # post save out
-    $self->GreenstoneXMLPlugout::post_saveas(@_);
-
-    # TODO: close database connection here? Or do something like with groups
-    # so we don't open and close over and over during a single build?
+    # 3. post save out
+    if($proc_mode ne "all" ) { # TODO Q
+	#$self->GreenstoneXMLPlugout::post_saveas(@_);
+	$self->GreenstoneXMLPlugout::post_saveas($doc_obj, $doc_dir, $docxml_outhandler, $output_file);
+    }
+    
+    # database connection is closed in end() method
+    # so we don't open and close over and over for each doc during a single build
 }
 
@@ -193,13 +224,4 @@
     my $doc_oid = $doc_obj->get_OID(); # we're processing a single doc at a time, so single OID
 
-    ##binmode($db_handle,":utf8"); ## WRONG FOR DB, NEED TO CREATE IN UTF8 MODE
-    
-    # TODO if $self->debug is on
-    
-    # Do we want to open and close a connection per doc?
-    # Would we not rather want to open and close per collection rebuild?
-    
-    #$self->create_db_connection();
-
     # load the prepared INSERT statement handles for both tables (can be undef for any table depending on whether meta_only or txt_only are set)
     my $metadata_table_sth = $self->{'metadata_prepared_insert_statement_handle'};
@@ -207,7 +229,6 @@
     
     $self->recursive_write_meta_and_text($doc_obj, $root_section, $metadata_table_sth, $fulltxt_table_sth);
-
-    #$self->close_db_connection();
-}
+}
+
 # Perl: Reading or Writing to Another Program
 # https://nnc3.com/mags/Perl3/cookbook/ch16_05.htm
@@ -290,297 +311,5 @@
 # Unless they do a full rebuild, which will recreate the table from scratch?
 # SOLUTION-> rollback transaction on error, see https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
-
-# I'm using perl's open2 like in Z3950Download, as opposed to open3 like in WgetDownload
-# since I'm assuming each insert statement is atomic: it either does the insertion or fails
-# and we (may) get some message back. That means we should hopefully be able to terminate
-# as well if we get SIGTERM/SIGKILL: we're not blocking, but are responsive after every INSERT stmt.
-
-# Just runs the command without displaying it
-sub run_command_basic
-{
-    my ($self,$strCMD) = @_; 
-    
-    
-    my $process_instr = $self->{'MYSQL_IN'};
-
-    print $process_instr "$strCMD\n";  
-}
-
-sub run_command
-{
-    my ($self,$strCMD) = @_;    
-
-    print STDERR "Running mysql command: $strCMD\n";
-    $self->run_command_basic($strCMD);
-    
-}
-
-sub response_line_contains {
-    my ($self,$expected) = @_;
-
-    my $out = $self->{'MYSQL_OUT'};
-    my $opening_line = <$out>;
-    if ($opening_line =~ m/$expected/i) {
-	return 1;
-    }
-    return 0;
-}
-
-sub response_lines_contain {
-    my ($self,$expected) = @_;
-
-    my $found_expected = 0;
-    my $out = $self->{'MYSQL_OUT'};
-    while (my $line = <$out>) {
-	print STDERR "$line\n";
-	if($line =~ m/$expected/){
-	    #return 1;
-	    $found_expected = 1; #won't break out of loop: loop will consume all on child out at present
-	}	
-    }
-    return $found_expected;
-}
-
-# based on Z3950Download.pm::start_yaz()
-# ./PATH/TO/mysql -u root -p
-# Returns:
-#   - 1 if client already quit after pwd fail (so can't send quit message to stopped client)
-#   - 0 if load_db failed (needs to be quit),
-#   - 1 if load_db succeeded (mysql client still running)
-sub create_db_connection {
-    my $self = shift (@_);
-    print STDERR "Opening connection to MySQL db\n";
-
-    my $mysql_client = $self->{'client_path'};
-    my $client_user = $self->{'client_user'} || "root";
-    
-    
-    my $launch_cmd = "\"./$mysql_client\" -u $client_user -p";
-    my $childpid = open2(*MYSQL_OUT, *MYSQL_IN, $launch_cmd)
-	or (print STDERR "Done\n" and die "can't open2 pipe to mysql client: $!");
-    
-    $self->{'pid'} = $childpid;
-    $self->{'MYSQL_OUT'} = *MYSQL_OUT;
-    $self->{'MYSQL_IN'} = *MYSQL_IN;
-    
-    # connect with pwd and load the database for this site
-    #my $conn_open = $self->open_connection();  
-    #
-    #if (!$conn_open) {
-    #    print STDERR "Cannot connect to mysql db with $launch_cmd\n"; 
-    #    print STDERR "Done\n";  
-    #    return 0;
-    #}
-    
-    # connect with pwd
-    my $conn_success = $self->send_pwd();    
-    if(!$conn_success) {
-	return -1; # if pwd failed, then the program already exited by itself
-	           # (so don't send quit command after process terminated)
-    }
-    else {
-	return $self->load_db();
-    }    
-
-    # return $conn_open;
-    return $conn_success; # 1 if client already quit after pwd fail, 0 if load_db failed (needs to be quit), 1 if load_db succeeded (mysql client still running)
-}
-
-
-# Copied from Z3950Download.pm::quit_yaz()
-sub close_db_connection {
-    my $self = shift (@_);
-    
-    $self->run_command("quit");
-    close($self->{'MYSQL_IN'}); # close the input to yaz. It also flushes quit command to mysql client
-
-    # make sure nothing is being output by mysql client
-    # flush the mysql-client process' outputstream, else we'll be stuck in an infinite
-    # loop waiting for the process to quit.
-    my $output = $self->{'MYSQL_OUT'};
-    my $line;
-    while (defined ($line = <$output>)) { 
-	if($line !~ m/\w/s) { # print anything other than plain whitespace in case it is important
-	    print STDERR "***### $line";
-	}
-    }
-
-    close($self->{'MYSQL_OUT'});
-
-    # Is the following necessary? The PerlDoc on open2 (http://perldoc.perl.org/IPC/Open2.html)
-    # says that waitpid must be called to "reap the child process", or otherwise it will hang
-    # around like a zombie process in the background. Adding it here makes the code work as 
-    # before, but it is certainly necessary to call waitpid on wget (see WgetDownload.pm).
-    # http://perldoc.perl.org/functions/waitpid.html
-    my $kidpid;
-    do {
-	$kidpid = waitpid($self->{'pid'}, WNOHANG);
-    } while $kidpid > 0; # waiting for pid to become -1
-}
-
-# should be called only once per site
-sub create_database {
-    my $self = shift (@_);
-    #my $sitename = shift(@_);
-    my $sitename = $self->{'site_name'};
-    my $cmd = "CREATE DATABASE $sitename;";
-    $self->run_command($cmd);
-}
-
-
-sub send_pwd {
-    my $self = shift (@_);
-
-    my $client_pwd = $self->{'client_pwd'};
-    my $out = $self->{'MYSQL_OUT'};
-    
-    # if connected, it's prompting for pwd. Write the pwd to the mysql client process:
-    $self->run_command_basic($client_pwd);
-    
-    #my $opening_line = <$out>;
-    #if ($opening_line =~ m/Access denied/i) {
-	#print STDERR "Password not recognised. Got: $opening_line\n";
-	#return 0;
-    #}
-    
-    if($self->response_line_contains("Access denied")) {
-	print STDERR "Password not recognised. Got: Access denied.\n";
-	return 0;
-    }
-
-    return 1;
-}
-
-
-sub load_db {
-    my $self = shift (@_);
-
-    # attempt to load the db
-    # use the database
-    my $db_name = $self->{'site_name'}; # TODO Q: site_name only exists for GS3. What about GS2?
-    $self->run_command("use " . $db_name . ";");
-    
-    my $db_found = 0;
-    my $out = $self->{'MYSQL_OUT'};
-    while (my $line = <$out>) {
-	print STDERR "$line\n";
-	if($line =~ m/Database changed/){
-	    # return 1; # TODO Q: consume all output of running command
-	    $db_found = 1;
-	}
-	elsif($line =~ m/Unknown database/){
-	    $db_found = -1;
-	}
-    }
-
-    if($db_found == 1) { 
-	return $db_found; 
-    }
-    elsif ($db_found == -1) { # a db for the current sitename didn't exist, create it
-	$self->create_database();
-
-	# attempt to load the newly created db
-	if($self->_load_db()) { # recursive call!
-	    return 1;
-	}
-	
-	
-	#my $opening_line = <$out>;
-	#if ($opening_line !~ m/Query OK/) {
-	if(!$self->response_line_contains("Query OK")) {
-	    print STDERR "Could not create db\n";
-	    return 0; # couldn't even create the db
-	} else { # success creating db
-	    # so let's create the metadata and fulltxt tables for the current coll while we're at it
-	    if($self->create_meta_table()) {	    
-		return $self->create_fulltxt_table();
-	    }
-	}
-    }
-    else { # unknown error trying to load db, bail
-	return 0;
-    }
-}
-
-
-sub create_meta_table {
-    my $self = shift (@_);
-    my $table_name = $self->{'collection_name'} . "metadata";
-    #my $cmd = "CREATE TABLE $table_name (id VARCHAR(255) NOT NULL UNIQUE, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, metaname VARCHAR(127) NOT NULL, metavalue VARCHAR(1023) NOT NULL, PRIMARY KEY(id));";
-
-    # If using an auto incremented primary key:
-    my $cmd = "CREATE TABLE $table_name (id INT NOT NULL AUTO_INCREMENT, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, metaname VARCHAR(127) NOT NULL, metavalue VARCHAR(1023) NOT NULL, PRIMARY KEY(id));";
-
-    $self->run_command($cmd);
-    
-    if(!$self->response_lines_contain("Query OK")) {
-	print STDERR "Could not create metadata table\n";
-	return 0;
-    } else {
-	return 1;
-    }
-}
-
-sub create_fulltxt_table {
-    my $self = shift (@_);
-    my $table_name = $self->{'collection_name'} . "fulltxt";
-    #my $cmd = "CREATE TABLE $table_name (id VARCHAR(255) NOT NULL UNIQUE, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, fulltxt LONGTEXT, PRIMARY KEY(id));";
-
-    # If using an auto incremented primary key:
-    my $cmd = "CREATE TABLE $table_name (id INT NOT NULL AUTO_INCREMENT, did VARCHAR(63) NOT NULL, sid VARCHAR(63) NOT NULL, fulltxt LONGTEXT, PRIMARY KEY(id));";
-
-    $self->run_command($cmd);
-    
-    if(!$self->response_lines_contain("Query OK")) {
-	print STDERR "Could not create table\n";
-	return 0;
-    } else {
-	return 1;
-    }
-}
-
-# https://www.guru99.com/insert-into.html
-# and https://dev.mysql.com/doc/refman/8.0/en/example-auto-increment.html
-#     for inserting multiple rows at once
-sub get_cmd_insert_new_row_into_meta_table {
-    my $self = shift (@_);
-    my ($did, $sid, $metaname, $metavalue) = @_;
-    my $tablename = $self->{'colname'}."_metadata";
-
-    my $cmd = "INSERT INTO $tablename (did, sid, metaname, metavalue) VALUES\n";
-    $cmd .= "('$did', '$sid', '$metaname', '$metavalue');\n";
-    return $cmd;
-}
-
-sub get_cmd_insert_new_row_into_txt_table {
-    my $self = shift (@_);
-    my ($did, $sid, $fulltext) = @_;
-    my $tablename = $self->{'colname'}."_fulltxt";
-
-    my $cmd = "INSERT INTO $tablename (did, sid, fulltxt) VALUES\n";
-    $cmd .= "('$did', '$sid', '$fulltext');\n";
-    return $cmd;
-}
-
-# TODO: later add edit and delete (and nothing for "been indexed" status?)
-
-################## 
-
-## UNUSED
-sub X_open_connection()
-{
-    my $self = shift (@_);
-    # connect with pwd
-    my $conn_success = $self->send_pwd();
-    if(!$conn_success) {
-	return -1; # if pwd failed, then the program already exists by itself (so don't send quit command after process terminated)
-    }
-    
-    if($conn_success) {
-	return $self->load_db();
-    } else {
-	return $conn_success;
-    }
-}
+# But then should set AutoCommit to off on connection, and remember to commit every time
 
 #################
@@ -598,6 +327,6 @@
     
     my $db_driver = $self->{'db_driver'};
-    my $db_user = $self->{'client_user'} || "root";
-    my $db_pwd = $self->{'client_pwd'};
+    my $db_user = $self->{'db_client_user'} || "root";
+    my $db_pwd = $self->{'db_client_pwd'};
     my $db_host = $self->{'db_host'} || "127.0.0.1";
     my $db_enc = $self->{'db_encoding'} || "utf8";
