Changeset 32544 for main/trunk

Timestamp:

2018-10-26T20:12:14+13:00 (5 years ago)

Author:

ak19

Message:

1. GreenstoneSQLPlugin: now sub read() calls the new lazy_get_gssql() method to only connect the first time and only if buildcol (when the processor var is a buildproc), so we don't connect in init() anymore as that got called during import.pl as well buildcol, whereas we only want to do connect/disconnet from DB stuff in this plugIN during buildcol.pl. 2. GSSQLPlugin and gssql now can delete entries from the SQL database. This can't be tested to work properly until GS SQL PlugOUT doesn't always assume removeold. At present it always does removeold: clearing the db even when incremental-import.pl is called. When there are no NEW docs in inc-import, the tables are cleared and nothing new is added, so can't test incremental behaviour during buildcol.pl/GS SQL PlugIN then either.

Location:

main/trunk/greenstone2/perllib

Files:

• gssql.pm (modified) (1 diff)
• plugins/GreenstoneSQLPlugin.pm (modified) (14 diffs)

main/trunk/greenstone2/perllib/gssql.pm

@@ -401 +401 @@
 }
 
+# delete all records in metatable with specified docid
+# https://www.tutorialspoint.com/mysql/mysql-delete-query.htm
+# DELETE FROM table_name [WHERE Clause]
+# see example under 'do' at https://metacpan.org/pod/release/TIMB/DBI-1.634_50/DBI.pm
+sub delete_recs_from_metatable_with_docid {
+    my $self= shift (@_);
+    my ($oid) = @_;
+    
+    my $dbh = $self->{'db_handle'};
+    my $meta_table = $self->get_metadata_table_name();
+    
+    #my $rows_deleted = 
+    $dbh->do(qq{DELETE FROM $meta_table WHERE did = ?}, undef, $oid)
+    or warn $dbh->errstr;
+}
+
+# delete all records in metatable with specified docid
+sub delete_recs_from_texttable_with_docid {
+    my $self= shift (@_);
+    my ($oid) = @_;
+    
+    my $dbh = $self->{'db_handle'};    
+    my $fulltxt_table = $self->get_fulltext_table_name();
+    
+    $dbh->do(qq{DELETE FROM $fulltxt_table WHERE did = ?}, undef, $oid)
+    or warn $dbh->errstr;
+}
 
 # Can call this after connection succeeded to get the database handle, dbh,

main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm

@@ -58 +58 @@
 # Discuss the plugin/plugout parameters.
 
+# TODO: when db is not running GLI is paralyzed -> can we set timeout on DBI connection attempt?
 
 # GreenstoneSQLPlugin inherits from GreenstoneXMLPlugin so that it if meta or fulltext
@@ -158 +159 @@
 
     # do anything else that needs to be done here when not pluginfo
+    #$self->{'delete_docids'} = (); # list of doc oids to delete during deinit()
     
     return $self;
@@ -179 +181 @@
     $self->{'doc_oid'} = $attr_hash{'docoid'};  
     print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
 
     }
@@ -196 +198 @@
     
     my $outhandle = $self->{'outhandle'};
-    my $doc_obj = $self->{'doc_obj'};    
-    my $gs_sql = $self->{'gs_sql'};
+    my $doc_obj = $self->{'doc_obj'};
+    # sub read() will make the db connection setting $self->{'gs_sql'} once: the first time read()
+    # is called on the GS SQLPlugin instance.
+    my $gs_sql = $self->{'gs_sql'} || return; # $self->lazy_get_gssql(); # won't want to call lazy_get_gssql() if close_doc called during (incr-)import.pl, only during buildcol.pl
+
+    # TODO: return statement skips "dummy" pass. Should we skip it or not?
+    # If we don't return, gs_sql is not set for dummy pass...
 
     my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
     print $outhandle "++++ OID of document (meta|text) to be read in from DB: $oid\n"
-    if $self->{'verbosity'} > 1;
+    if $self->{'verbosity'} > 2;
     
     # For now, we have access to doc_obj (until just before super::close_document() terminates)
@@ -214 +221 @@
     # read in meta for the collection (i.e. select * from <col>_metadata table
 
-    my $sth = $gs_sql->select_from_metatable_matching_docid($oid);  
+    my $sth = $gs_sql->select_from_metatable_matching_docid($oid);
     print $outhandle "### SQL select stmt: ".$sth->{'Statement'}."\n"
-        if $self->{'verbosity'} > 1;
-
-    print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
+
+    print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
     # https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
     while( my @row = $sth->fetchrow_array() ) {     
@@ -228 +235 @@
         $sid = $doc_obj->get_top_section() unless $sid;
         print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
         
         # TODO:  we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_meta directly:
@@ -234 +241 @@
     }
     print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
     }
     
@@ -244 +251 @@
     
     my $sth = $gs_sql->select_from_texttable_matching_docid($oid);
-    print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 1;
+    print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 2;
 
     print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
     while( my ($primary_key, $did, $sid, $text) = $sth->fetchrow_array() ) {       
 
@@ -254 +261 @@
         $sid = $doc_obj->get_top_section() if ($sid eq "root");
         print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
 
         # TODO - pass by ref?
@@ -261 +268 @@
     }   
     print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
-        if $self->{'verbosity'} > 1;
+        if $self->{'verbosity'} > 2;
     }
 
@@ -274 +281 @@
 # During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
 # effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
+# Lazy connection.
 
 # Call init() not begin() because there can be multiple plugin passes
@@ -279 +287 @@
 # one for doc level and another for section level indexing
 # This way, we can connect to the SQL database once per buildcol run.
-sub init {
-    my ($self) = shift (@_);
+#sub init {
+#    my ($self) = shift (@_);
 #    print STDERR "@@@@@@@@@@ INIT CALLED\n";
     
-    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
-
+#    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
+
+
+sub lazy_get_gssql {
+    my $self = shift(@_);
+
+    # if we failed to successfully connect once before, don't bother attempting to connect again
+    #return undef if(defined $self->{'failed'}); # plugin/process would have terminated with die()
+                                  # if we couldn't succeed connecting on any connection attempt
+    
+    return $self->{'gs_sql'} if($self->{'gs_sql'});
+
+    # assume we'll fail to connect
+    $self->{'failed'} = 1;
+
+    print STDERR "@@@@@@@@@@ LAZY CONNECT CALLED\n";
+    
     ####################
 #    print "@@@ SITE NAME: ". $self->{'site_name'} . "\n" if defined $self->{'site_name'};
@@ -328 +351 @@
     die("Could not use db $db_name. Can't proceed.\n");
     }
+
+    #undef $self->{'failed'};
     
     # store db handle now that we're connected
     $self->{'gs_sql'} = $gs_sql;
+    return $gs_sql;
     
 }
@@ -341 +367 @@
 sub deinit {
     my ($self) = shift (@_);
-    if($self->{'gs_sql'}) { # can cover TODO: only want to work with sql db if buildcol.pl
+    
+    print STDERR "@@@@@@@@@@ GreenstoneSQLPlugin::DEINIT CALLED\n";
+    
+    if($self->{'gs_sql'}) { # only want to work with sql db if buildcol.pl, gs_sql won't have
+    # a value except during buildcol, so when processor =~ m/buildproc$/.
     $self->{'gs_sql'}->disconnect_from_db()
         || warn("Unable to disconnect from database " . $self->{'site_name'} . "\n");
-    }
-#    print STDERR "@@@@@@@@@@ DEINIT CALLED\n";
+
+    # explicitly set to undef so all future use has to make the connection again
+    undef $self->{'gs_sql'};
+    }
+
     $self->SUPER::deinit(@_);
 }
 
+# TODO: This can't work until GSSQLPlugout has implemented build_mode = incremental
+# (instead of tossing away db on every build)
+# then this method needs to undef $self->docid after deleting, and close_doc() has to
+# just return if $self->docid undefined
+
+sub read {
+    my $self = shift (@_);  
+  
+    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
+
+    my $rv = $self->SUPER::read(@_); # defined in ReadXMLFile inherited by superclass GS XML PLugin
+
+    if(defined $rv) { # undef if !can_proc_this_file, but -1 if failed to parse docsql.xml
+
+    # don't want to do any GreenstoneSQLPlugin DB stuff during import.pl
+    # only during in buildcol.pl    
+    return if (ref($processor) !~ m/buildproc$/i);
+
+    # we know we're buildcol, let's proceed:
+
+    # make the connection once for the life of the plugin, not once for every doc
+    # so that we can disconnect at the very end of the plugin's life: on deinit()
+    # If we hadn't connected before, connect now
+    my $gs_sql = $self->{'gs_sql'} || $self->lazy_get_gssql(); # TODO which syntax best?
+    
+    my $build_proc_mode = $processor->get_mode(); # can be "text" as per basebuildproc or
+    # "textdelete" or "textreindex" as per ArchivesInfPlugin
+    if($build_proc_mode =~ m/\.delete/) {
+
+        # NOTTODO: add current doc OID stored in $self->{'doc_oid'} to list of oids get rid 
+        # of from table(s) entries. We'll do the actual deletion in deinit?? Since that's
+        # when ArchivesInfPlugin deletes the docsql.xml files
+        
+        my $doc_oid = $self->{'doc_oid'};
+        #my @delete_docids = $self->{'delete_docids'};
+        #push (@delete_docids, $doc_oid);
+
+        my $proc_mode = $self->{'process_mode'};
+        if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+        print STDERR "@@@@@@@@ Deleting $doc_oid from meta table\n";
+        $gs_sql->delete_recs_from_metatable_with_docid($doc_oid);
+        }
+        if($proc_mode eq "all" || $proc_mode eq "text_only") {
+        print STDERR "@@@@@@@@ Deleting $doc_oid from fulltxt table\n";
+        $gs_sql->delete_recs_from_texttable_with_docid($doc_oid);
+        }
+    }
+    }
+
+    return $rv;
+    
+}
+