Changeset 32555 for main/trunk/greenstone2

Timestamp:

2018-10-30T19:29:56+13:00 (5 years ago)

Author:

ak19

Message:

1. In GreenstoneSQLPlugout, removeold is now paramterised (as are keepold, incremental, incremental_mode). 2. Deletion on incremental_build works. But there are more questions. Why are there 4 passes? What to do on reindexing and when to do it (should it happen during GS SQL plugout or plugin)?

Location:

main/trunk/greenstone2/perllib

Files:

• gssql.pm (modified) (1 diff)
• inexport.pm (modified) (1 diff)
• plugins/GreenstoneSQLPlugin.pm (modified) (9 diffs)
• plugouts/BasePlugout.pm (modified) (1 diff)
• plugouts/GreenstoneSQLPlugout.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/gssql.pm

@@ -186 +186 @@
     
 
+    # build_mode can be removeold or incremental. We only do something special on removeold:
+    # deleting the existing tables for this collection and recreating empty ones
     if($build_mode eq "removeold") {
         $self->delete_collection_tables();

main/trunk/greenstone2/perllib/inexport.pm

@@ -690 +690 @@
     }
     my $processor = &plugout::load_plugout($plugout);
+    $processor->set_incremental_options($removeold, $keepold, $incremental, $incremental_mode);
     $processor->setoutputdir ($archivedir);
     $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;

main/trunk/greenstone2/perllib/plugins/GreenstoneSQLPlugin.pm

@@ -40 +40 @@
 # TODO:
 # - Run TODOs here, in Plugout and in gssql.pm by Dr Bainbridge.
+# - "Courier" demo documents in lucene-sql collection: character (degree symbol) not preserved in title. Is this because we encode in utf8 when putting into db and reading back in?
+# Test doc with meta and text like macron in Maori text.
+# - Have not yet tested writing out just meta or just fulltxt to sql db and reading just that 
+# back in from the sql db while the remainder is to be read back in from the docsql .xml files.
+
+# TODO: deal with incremental vs removeold. If docs removed from import folder, then import step
+# won't delete it from archives but buildcol step will. Need to implement this with this database plugin or wherever the actual flow is
+
+# TODO: Add public instructions on using this plugin and its plugout: start with installing mysql binary, changing pwd, running the server (and the client against it for checking, basic cmds like create and drop). Then discuss db name, table names (per coll), db cols and col types, and how the plugout and plugin work.
+# Discuss the plugin/plugout parameters.
+
+# TODO: when db is not running GLI is paralyzed -> can we set timeout on DBI connection attempt?
+
+# TODO Q: is "reindex" = del from db + add to db?
+# - is this okay for reindexing, or will it need to modify existing values (update table)
+# - if it's okay, what does reindex need to accomplish (and how) if the OID changes because hash id produced is different?
+# - delete is accomplished in GS SQL Plugin, during buildcol.pl. When should reindexing take place?
+# during SQL plugout/import.pl or during plugin? If adding is done by GSSQLPlugout, does it need to
+# be reimplemented in GSSQLPlugin to support the adding portion of reindexing.
+
+
+# TODO Q: During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
+# effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
+
+
+# + TODO: Incremental delete can't work until GSSQLPlugout has implemented build_mode = incremental
+# (instead of tossing away db on every build)
 # + Ask about docsql naming convention adopted to identify OID. Better way?
 # collection names -> table names: it seems hyphens not allowed. Changed to underscores.
 # + Startup parameters (except removeold/build_mode)
-# - how do we detect we're to do removeold during plugout in import.pl phase
-# - incremental building: where do we need to add code to delete rows from our sql table after
+# + how do we detect we're to do removeold during plugout in import.pl phase
+# + incremental building: where do we need to add code to delete rows from our sql table after
 # incrementally importing a coll with fewer docs (for instance)? What about deleted/modified meta?
-# - "Courier" demo documents in lucene-sql collection: character (degree symbol) not preserved in title. Is this because we encode in utf8 when putting into db and reading back in?
-# - Have not yet tested writing out just meta or just fulltxt to sql db and reading just that 
-# back in from the sql db while the remainder is to be read back in from the docsql .xml files.
-# - Ask if I can assume that all SQL dbs (not just MySQL) will preserve the order of inserted nodes
-# (sections) which in this case had made it easy to reconstruct the doc_obj in memory in the correct order
-
-# TODO: deal with incremental vs removeold. If docs removed from import folder, then import step
-# won't delete it from archives but buildcol step will. Need to implement this with this database plugin or wherever the actual flow is
-
-# TODO: Add public instructions on using this plugin and its plugout: start with installing mysql binary, changing pwd, running the server (and the client against it for checking, basic cmds like create and drop). Then discuss db name, table names (per coll), db cols and col types, and how the plugout and plugin work.
-# Discuss the plugin/plugout parameters.
-
-# TODO: when db is not running GLI is paralyzed -> can we set timeout on DBI connection attempt?
+# + Ask if I can assume that all SQL dbs (not just MySQL) will preserve the order of inserted nodes
+# (sections) which in this case had made it easy to reconstruct the doc_obj in memory in the correct order.
+# YES: Otherwise for later db types (drivers), can set order by primary key column and then order by did column
+
+
+########################################################################################
 
 # GreenstoneSQLPlugin inherits from GreenstoneXMLPlugin so that it if meta or fulltext
@@ -179 +199 @@
     # whereas $_ has the tag info. So we don't want to do $_{'docoid'}.
     my %attr_hash = %_; # right way, see OAIPlugin.pm
-    $self->{'doc_oid'} = $attr_hash{'docoid'};  
+    $self->{'doc_oid'} = $attr_hash{'docoid'};
+    print STDERR "XXXXXXXXXXXXXX in SQLPlugin::xml_start_tag()\n";
     print $outhandle "Extracted OID from docsql.xml: ".$self->{'doc_oid'}."\n"
         if $self->{'verbosity'} > 2;
@@ -189 +210 @@
 }
 
-# TODO Q: Why are there 3 passes when we're only indexing at doc and section level (2 passes)?
+# TODO Q: Why are there 4 passes when we're only indexing at doc and section level (2 passes)? What's the dummy pass, why is there a pass for infodb?
 
 # At the end of superclass GreenstoneXMLPlugin.pm's close_document() method,
@@ -196 +217 @@
 sub close_document {
     my $self = shift(@_);
+
+    print STDERR "XXXXXXXXX in SQLPlugin::close_doc()\n";
+    
+    my $gs_sql = $self->get_gssql_instance(); 
     
     my $outhandle = $self->{'outhandle'};
     my $doc_obj = $self->{'doc_obj'};
-    # sub read() will make the db connection setting $self->{'gs_sql'} once: the first time read()
-    # is called on the GS SQLPlugin instance.
-    my $gs_sql = $self->{'gs_sql'} || return; # $self->lazy_get_gssql(); # won't want to call lazy_get_gssql() if close_doc called during (incr-)import.pl, only during buildcol.pl
-
-    # TODO: return statement skips "dummy" pass. Should we skip it or not?
-    # If we don't return, gs_sql is not set for dummy pass...
-
+
+    my $build_proc_mode = $self->{'processor'}->get_mode(); # can be "text" as per basebuildproc or infodb
     my $oid = $self->{'doc_oid'}; # we stored current doc's OID during sub xml_start_tag()
-    print $outhandle "++++ OID of document (meta|text) to be read in from DB: $oid\n"
+    my $proc_mode = $self->{'process_mode'};
+    
+    print $outhandle "++++ OID of document (meta|text) to be del or read in from DB: ".$self->{'doc_oid'}."\n"
     if $self->{'verbosity'} > 2;
     
     # For now, we have access to doc_obj (until just before super::close_document() terminates)
-
+    
     # no need to call $self->{'doc_obj'}->set_OID($oid);
     # because either the OID is stored in the SQL db as meta 'Identifier' alongside other metadata
     # or it's stored in the doc.xml as metadata 'Identifier' alongside other metadata
     # Either way, Identifier meta will be read into the docobj automatically with other meta.
-
-    my $proc_mode = $self->{'process_mode'};
-    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
-    # read in meta for the collection (i.e. select * from <col>_metadata table
-
-    my $sth = $gs_sql->select_from_metatable_matching_docid($oid);
-    print $outhandle "### SQL select stmt: ".$sth->{'Statement'}."\n"
-        if $self->{'verbosity'} > 2;
-
-    print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
-    # https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
-    while( my @row = $sth->fetchrow_array() ) {     
-        #print $outhandle "row: @row\n";
-        my ($primary_key, $did, $sid, $metaname, $metaval) = @row;
-
-        # get rid of the artificial "root" introduced in section id when saving to sql db
-        $sid =~ s@^root@@;
-        $sid = $doc_obj->get_top_section() unless $sid;
-        print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
+    
+    if ($self->{'verbosity'} > 2) {
+    print STDERR "+++++++++++ buildproc_mode: $build_proc_mode\n";
+    print STDERR "+++++++++++ SQLPlug proc_mode: $proc_mode\n";
+    }
+
+    # TODO: where does reindexing take place, GreenstoneSQL -Plugout or -Plugin?
+    #if($build_proc_mode =~ m/(delete|reindex)$/) { # doc denoted by current OID has been marked for deletion or reindexing (=delete + add)
+    if($build_proc_mode =~ m/(delete)$/) { # doc denoted by current OID has been marked for deletion or reindexing (=delete + add)
+    
+          # build_proc_mode could be "(infodb|text)(delete|reindex)"
+          # "...delete" or "...reindex" as per ArchivesInfPlugin
+    
+    print STDERR "@@@@ DELETING DOC FROM SQL DB\n";
+    
+    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+        print STDERR "@@@@@@@@ Deleting $oid from meta table\n" if $self->{'verbosity'} > 2;
+        $gs_sql->delete_recs_from_metatable_with_docid($oid);
+    }
+    if($proc_mode eq "all" || $proc_mode eq "text_only") {
+        print STDERR "@@@@@@@@ Deleting $oid from fulltxt table\n" if $self->{'verbosity'} > 2;
+        $gs_sql->delete_recs_from_texttable_with_docid($oid);
+    }
+    
+    # If we're reindexing the current doc, we will we want to continue: which
+    # will add this doc ID back into the db with the new meta/full txt values
+    # But if we're deleting, then we're done processing the document, so set doc_oid to undef
+    # to prevent adding it back into db
+    #undef $self->{'doc_oid'} if($build_proc_mode =~ m/delete$/);   
+    
+    } # done deleting doc from SQL db    
+    
+    else {#if($self->{'doc_oid'}) { # if loading doc from SQL db
+    print STDERR "@@@@ LOADING DOC FROM SQL DB\n";  
+    
+    if($proc_mode eq "all" || $proc_mode eq "meta_only") {
+        # read in meta for the collection (i.e. select * from <col>_metadata table
+        
+        my $sth = $gs_sql->select_from_metatable_matching_docid($oid);
+        print $outhandle "### SQL select stmt: ".$sth->{'Statement'}."\n"
         if $self->{'verbosity'} > 2;
         
-        # TODO:  we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_meta directly:
-        $doc_obj->add_utf8_metadata($sid, $metaname, &docprint::unescape_text($metaval));
+        print $outhandle "----------SQL DB contains meta-----------\n" if $self->{'verbosity'} > 2;
+        # https://www.effectiveperlprogramming.com/2010/07/set-custom-dbi-error-handlers/
+        while( my @row = $sth->fetchrow_array() ) {     
+        #print $outhandle "row: @row\n";
+        my ($primary_key, $did, $sid, $metaname, $metaval) = @row;
+        
+        # get rid of the artificial "root" introduced in section id when saving to sql db
+        $sid =~ s@^root@@;
+        $sid = $doc_obj->get_top_section() unless $sid;
+        print $outhandle "### did: $did, sid: |$sid|, meta: $metaname, val: $metaval\n"
+            if $self->{'verbosity'} > 2;
+        
+        # TODO:  we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_meta directly:
+        $doc_obj->add_utf8_metadata($sid, $metaname, &docprint::unescape_text($metaval));
+        }
+        print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
+        if $self->{'verbosity'} > 2;
     }
-    print $outhandle "----------FIN READING DOC's META FROM SQL DB------------\n"
-        if $self->{'verbosity'} > 2;
-    }
-    
-    if($proc_mode eq "all" || $proc_mode eq "text_only") {
-    # read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
-
-    my $fulltxt_table = $gs_sql->get_fulltext_table_name();
-    
-    
-    my $sth = $gs_sql->select_from_texttable_matching_docid($oid);
-    print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 2;
-
-    print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
-        if $self->{'verbosity'} > 2;
-    while( my ($primary_key, $did, $sid, $text) = $sth->fetchrow_array() ) {       
-
-        # get rid of the artificial "root" introduced in section id when saving to sql db
-        #$sid =~ s@^root@@;
-        $sid = $doc_obj->get_top_section() if ($sid eq "root");
-        print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
+    
+    if($proc_mode eq "all" || $proc_mode eq "text_only") {
+        # read in fulltxt for the collection (i.e. select * from <col>_fulltxt table
+        
+        my $fulltxt_table = $gs_sql->get_fulltext_table_name();
+        
+    
+        my $sth = $gs_sql->select_from_texttable_matching_docid($oid);
+        print $outhandle "### stmt: ".$sth->{'Statement'}."\n" if $self->{'verbosity'} > 2;
+        
+        print $outhandle "----------\nSQL DB contains txt entries for-----------\n"
         if $self->{'verbosity'} > 2;
-
-        # TODO - pass by ref?
-        # TODO: we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_text directly:
-        $doc_obj->add_utf8_text($sid, &docprint::unescape_text($text));
-    }   
-    print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
-        if $self->{'verbosity'} > 2;
-    }
-
+        while( my ($primary_key, $did, $sid, $text) = $sth->fetchrow_array() ) {       
+        
+        # get rid of the artificial "root" introduced in section id when saving to sql db
+        #$sid =~ s@^root@@;
+        $sid = $doc_obj->get_top_section() if ($sid eq "root");
+        print $outhandle "### did: $did, sid: |$sid|, fulltext: <TXT>\n"
+            if $self->{'verbosity'} > 2;
+        
+        # TODO - pass by ref?
+        # TODO: we accessed the db in utf8 mode, so, we can call doc_obj->add_utf8_text directly:
+        $doc_obj->add_utf8_text($sid, &docprint::unescape_text($text));
+        }   
+        print $outhandle "----------FIN READING DOC's TXT FROM SQL DB------------\n"
+        if $self->{'verbosity'} > 2;
+    }
+    
+    } # done reading into docobj from SQL db
     
     # don't forget to clean up on close() in superclass
@@ -278 +334 @@
 
 
-# TODO: only want to work with sql db if buildcol.pl. Unfortunately, also runs on import.pl.
-# During import, the GS SQL Plugin is called before the GS SQL Plugout with undesirable side
-# effect that if the db doesn't exist, gssql::use_db() fails, as it won't create db.
-# Lazy connection.
-
-# Call init() not begin() because there can be multiple plugin passes
-# and init() should be called before all passes:
-# one for doc level and another for section level indexing
-# This way, we can connect to the SQL database once per buildcol run.
-#sub init {
-#    my ($self) = shift (@_);
-#    print STDERR "@@@@@@@@@@ INIT CALLED\n";
-    
-#    $self->SUPER::init(@_); # super (GreenstoneXMLPlugin) will not yet be trying to read from doc.xml (docsql .xml) files in init().
-
-
-sub lazy_get_gssql {
+# We want SQLPlugin to connect to db only during buildcol.pl phase, not during import.pl
+# This works out okay, as close_document() (called by read()) is only invoked during buildcol.pl
+#
+# Further, we want a single db connection for the GS SQL Plugin to be used for 
+# the multiple plugin passes: for "dummy" pass, and for doc level and for section level indexing
+# By calling the lazy loading get_sql_instance() from close_document(),
+# we connect to the SQL database once per GSSQLPlugin and only during the buildcol phase.
+#
+# get_gssql_instance() is a lazy loading method that returns singleton db connection for a GreenstoneSQLPlugin object. ("Code pattern" get instance vs singleton.)
+# One instance of db connection that can be used for all the many doc_objects processed by this plugin
+#
+# Except in methods get_gssql_instance() and deinit(), don't access self->{'_gs_sql'} directly.
+# Instead, call method get_gssql_instance() and store return value in a local variable, my $gs_sql
+#
+sub get_gssql_instance
+{    
     my $self = shift(@_);
 
@@ -301 +356 @@
                                   # if we couldn't succeed connecting on any connection attempt
     
-    return $self->{'gs_sql'} if($self->{'gs_sql'});
+    return $self->{'_gs_sql'} if($self->{'_gs_sql'});
 
     # assume we'll fail to connect
@@ -355 +410 @@
     
     # store db handle now that we're connected
-    $self->{'gs_sql'} = $gs_sql;
+    $self->{'_gs_sql'} = $gs_sql;
     return $gs_sql;
     
@@ -370 +425 @@
     print STDERR "@@@@@@@@@@ GreenstoneSQLPlugin::DEINIT CALLED\n";
     
-    if($self->{'gs_sql'}) { # only want to work with sql db if buildcol.pl, gs_sql won't have
+    if($self->{'_gs_sql'}) { # only want to work with sql db if buildcol.pl, gs_sql won't have
     # a value except during buildcol, so when processor =~ m/buildproc$/.
-    $self->{'gs_sql'}->disconnect_from_db()
+    $self->{'_gs_sql'}->disconnect_from_db()
         || warn("Unable to disconnect from database " . $self->{'site_name'} . "\n");
 
     # explicitly set to undef so all future use has to make the connection again
-    undef $self->{'gs_sql'};
+    undef $self->{'_gs_sql'};
     }
 
@@ -382 +437 @@
 }
 
-# TODO: This can't work until GSSQLPlugout has implemented build_mode = incremental
-# (instead of tossing away db on every build)
-# then this method needs to undef $self->docid after deleting, and close_doc() has to
-# just return if $self->docid undefined
-
-sub read {
-    my $self = shift (@_);  
-  
-    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli) = @_;
-
-    my $rv = $self->SUPER::read(@_); # defined in ReadXMLFile inherited by superclass GS XML PLugin
-
-    if(defined $rv) { # undef if !can_proc_this_file, but -1 if failed to parse docsql.xml
-
-    # don't want to do any GreenstoneSQLPlugin DB stuff during import.pl
-    # only during in buildcol.pl    
-    return if (ref($processor) !~ m/buildproc$/i);
-
-    # we know we're buildcol, let's proceed:
-
-    # make the connection once for the life of the plugin, not once for every doc
-    # so that we can disconnect at the very end of the plugin's life: on deinit()
-    # If we hadn't connected before, connect now
-    my $gs_sql = $self->{'gs_sql'} || $self->lazy_get_gssql(); # TODO which syntax best?
-    
-    my $build_proc_mode = $processor->get_mode(); # can be "text" as per basebuildproc or
-    # "textdelete" or "textreindex" as per ArchivesInfPlugin
-    if($build_proc_mode =~ m/\.delete/) {
-
-        # NOTTODO: add current doc OID stored in $self->{'doc_oid'} to list of oids get rid 
-        # of from table(s) entries. We'll do the actual deletion in deinit?? Since that's
-        # when ArchivesInfPlugin deletes the docsql.xml files
-        
-        my $doc_oid = $self->{'doc_oid'};
-        #my @delete_docids = $self->{'delete_docids'};
-        #push (@delete_docids, $doc_oid);
-
-        my $proc_mode = $self->{'process_mode'};
-        if($proc_mode eq "all" || $proc_mode eq "meta_only") {
-        print STDERR "@@@@@@@@ Deleting $doc_oid from meta table\n";
-        $gs_sql->delete_recs_from_metatable_with_docid($doc_oid);
-        }
-        if($proc_mode eq "all" || $proc_mode eq "text_only") {
-        print STDERR "@@@@@@@@ Deleting $doc_oid from fulltxt table\n";
-        $gs_sql->delete_recs_from_texttable_with_docid($doc_oid);
-        }
-    }
-    }
-
-    return $rv;
-    
-}
-
+
+
+

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -347 +347 @@
 }
 
+# GreenstoneSQLPlugout needs to know whether we're doing removeold or not
+sub set_incremental_options {
+    my $self= shift (@_);
+    my ($removeold, $keepold, $incremental, $incremental_mode) = @_;
+    
+    $self->{'removeold'}        = $removeold;
+    $self->{'keepold'}          = $keepold;
+    $self->{'incremental'}      = $incremental;
+    $self->{'incremental_mode'} = $incremental_mode;
+}
+
 # OIDtype may be "hash" or "hash_on_full_filename" or "incremental" or "filename" or "dirname" or "full_filename" or "assigned"
 sub set_OIDtype {

main/trunk/greenstone2/perllib/plugouts/GreenstoneSQLPlugout.pm

@@ -139 +139 @@
     #print STDERR "@@@@ db_driver: " . $self->{'db_driver'} . "\n";
     #print STDERR "@@@@ proc_mode: " . $self->{'process_mode'} . "\n";
-    
-    ########### TODO: deal with build mode #########
-
-    $self->{'build_mode'} = "removeold";
 
     ############ LOAD NECESSARY OPTIONS ###########
@@ -173 +169 @@
     
     my $db_name = $self->{'site_name'} || "greenstone2"; # one database per GS3 site, for GS2 the db is called greenstone2
-    my $build_mode = $self->{'build_mode'} || "removeold";
+    my $build_mode = ($self->{'removeold'}) ? "removeold" : "incremental";
+    print STDERR "@@@@@@@@@@@@ remove_old: $build_mode\n";
+    
     if(!$gs_sql->load_db_and_tables($db_name, $build_mode)) {