Changeset 23133 for main/trunk/greenstone2/perllib/basebuildproc.pm

Timestamp:

2010-10-13T14:25:54+13:00 (14 years ago)

Author:

kjdon

Message:

still workign on incremental infodb updating. cleaning up code now that reconstructed docs are added after processing new/changed ones. so don't need to do deletion from the infodb.

File:

• main/trunk/greenstone2/perllib/basebuildproc.pm (modified) (7 diffs)

main/trunk/greenstone2/perllib/basebuildproc.pm

@@ -148 +148 @@
     # reconstructed docs have no text, just metadata, so we need to 
     # remember how many bytes we had initially
-    $self->{'num_bytes'}     = $self->{'starting_num_bytes'};
-    
+    #$self->{'num_bytes'}     = $self->{'starting_num_bytes'};
+    $self->{'num_bytes'} = 0; # we'll store num bytes in db for reconstructed docs.
     $self->{'num_processed_bytes'} = 0;
 }
@@ -389 +389 @@
     my $self = shift (@_);
     my ($doc_obj, $filename, $edit_mode) = @_;
-
+    
     # only output this document if it is a "indexed_doc" or "info_doc" (database only) document
     my $doctype = $doc_obj->get_doc_type();
     return if ($doctype ne "indexed_doc" && $doctype ne "info_doc");
-
+    print STDERR "infodbedit, mode=$edit_mode, ".$doc_obj->get_OID()."\n";
     my $archivedir = "";
     if (defined $filename)
@@ -416 +416 @@
     }
 
-    if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-    #add this document to the browse structure
-    push(@{$self->{'doclist'}},$doc_obj->get_OID()) 
-        unless ($doctype eq "classification");
-    if ($edit_mode eq "update") {
-        $self->{'dont_reconstruct'}->{$doc_obj->get_OID()} = 1;
-    }
-    &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
-    $self->{'num_docs'} += 1 unless ($doctype eq "classification");
-    }
-    elsif ($edit_mode eq "delete") {
+    if ($edit_mode eq "delete") {
     # record this doc so we don't process the reconstructed doc later
-    my $del_doc_oid = $doc_obj->get_OID();
-    $self->{'dont_reconstruct'}->{$del_doc_oid} = 1;
+    $self->{'dont_reconstruct'}->{$doc_obj->get_OID()} = 1;
+    # we don't need to do anything else for the info database for a deleted document. The infodb starts from scratch each time, so no deletion is necessary
     # do we need this??? where did num_docs come from, from reconstruction??
-    $self->{'num_docs'} -= 1 unless ($doctype eq "classification");
-    }
-
+    #$self->{'num_docs'} -= 1 unless ($doctype eq "classification");
+    return;
+    }
+    
+    if ($edit_mode eq "update") {
+    # we don't want to process the reconstructed doc later, but we will process this version now.
+    $self->{'dont_reconstruct'}->{$doc_obj->get_OID()} = 1;
+    }
+
+    # rest of code used for add and update. In both cases, we add to the classifiers and to the info database. 
+
+    #add this document to the browse structure
+    push(@{$self->{'doclist'}},$doc_obj->get_OID()) 
+    unless ($doctype eq "classification");
+    $self->{'num_docs'} += 1 unless ($doctype eq "classification");
+    
+    if (!defined $filename) {
+    # a reconstructed doc
+    $self->{'num_bytes'} += $doc_obj->get_metadata_element ($doc_obj->get_top_section (), "total_numbytes");
+    print STDERR "new numbytes = $self->{'num_bytes'}\n";
+    }
+    # classify the document
+    &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
+    
+    # now add all the section to the infodb.
+    
     # is this a paged or a hierarchical document
     my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
@@ -453 +466 @@
     my %section_infodb = ();
 
-    # update a few statistics - how do we do update????
-    if (($edit_mode eq "add")) {# || ($edit_mode eq "update")) {
-
-        $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
-        $self->{'num_sections'} += 1 unless ($doctype eq "classification");
-    }
-    elsif ($edit_mode eq "delete") {
-        # delete
-        $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
-        $self->{'num_sections'} -= 1 unless ($doctype eq "classification");
-    }
-
+    # update a few statistics 
+    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+    $self->{'num_sections'} += 1 unless ($doctype eq "classification");
+    
     # output the fact that this document is a document (unless doctype
     # has been set to something else from within a plugin
@@ -472 +477 @@
     }
 
+    if ($first && defined $filename) {
+        # if we are at the top level of the document, and we are not a reconstructed document, set the total_text_length - used to count bytes when we reconstruct later
+        my $length = $doc_obj->get_total_text_length();
+        $section_infodb{"total_numbytes"} = [ $length ];
+    }
     # Output whether this node contains text
     #
@@ -504 +514 @@
         # special case for URL metadata
         if ($field =~ /^URL$/i) {
-            if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-
-            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value, { 'section' => [ $section_OID ] });
-            }
-            else {
-            # delete
-            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value);
-            }
-
-            
+            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $value, { 'section' => [ $section_OID ] });
         }
-
+        
         if (!defined $self->{'dontdb'}->{$field}) {
             push(@{$section_infodb{$field}}, $value);
@@ -590 +591 @@
     } 
     
-    if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-        # in case of update, this will overwrite old entry??
-        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID, \%section_infodb);
-    }
-    else {
-        # delete
-        &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID);
-    }
-
-    
-    # output a database entry for the document number, except for Lucene (which no longer needs this information)
-    unless (ref($self) eq "lucenebuildproc")
+    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $section_OID, \%section_infodb);
+        
+    # output a database entry for the document number, unless we are incremental
+    unless ($self->is_incremental_capable())
     {
-        if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-        
-        if ($self->{'db_level'} eq "document") {
-            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'}, { 'section' => [ $doc_OID ] });
-        }
-        else {
-            &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'}, { 'section' => [ $section_OID ] });
-        }
+        print STDERR "outputting db entry for doc number\n";
+        if ($self->{'db_level'} eq "document") {
+        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'}, { 'section' => [ $doc_OID ] });
         }
         else {
-
-        if ($self->{'db_level'} eq "document") {
-            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_docs'});
-        }
-        else {
-            &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'});
-        }
-
-        }
-    }
-
+        &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_handle, $self->{'num_sections'}, { 'section' => [ $section_OID ] });
+        }
+    }
+        
     $first = 0;
     $section = $doc_obj->get_next_section($section);
     last if ($self->{'db_level'} eq "document"); # if no sections wanted, only add the docs
-    }
+    } # while defined section
+
+    print STDERR "end of infodb edit\n";
 }