Changeset 18456 for gsdl/trunk/perllib/lucenebuildproc.pm

Timestamp:

2009-02-03T09:48:19+13:00 (15 years ago)

Author:

davidb

Message:

Additions to support the deleting of documents from the index. Only works for indexers that support incremental building, e.g. lucene

File:

• gsdl/trunk/perllib/lucenebuildproc.pm (modified) (10 diffs)

gsdl/trunk/perllib/lucenebuildproc.pm

@@ -63 +63 @@
 
 
-sub text {
-    my $self = shift (@_);
-    my ($doc_obj,$file) = @_;
-    my $handle = $self->{'output_handle'};
+sub textedit {
+    my $self = shift (@_);
+    my ($doc_obj,$file,$edit_mode) = @_;
+
+    my $lucenehandle = $self->{'output_handle'};
     my $outhandle = $self->{'outhandle'};
 
@@ -72 +73 @@
     return if ($doc_obj->get_doc_type() ne "indexed_doc");
 
+    # skip this document if in "compress-text" mode and asked to delete it
+    return if (!$self->get_indexing_text() && ($edit_mode eq "delete"));
+
     my $indexed_doc = $self->is_subcollection_doc($doc_obj);
 
     # this is another document
-    $self->{'num_docs'} += 1;
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+    $self->{'num_docs'} += 1;
+    }
+    else {
+    $self->{'num_docs'} -= 1;
+    }
 
     # get the parameters for the output
@@ -86 +95 @@
     my $ldoc_level = $levels->{'document'};
     my $lsec_level = $levels->{'section'};
-    #my $lpar_level = $levels->{'paragraph'};
-
+
+    # gs2_id should be depricated #####
     my $gs2_id = "";
     if ($ldoc_level)
@@ -102 +111 @@
     }
     my $gs2_docOID = $doc_obj->get_OID();
-    my $documenttag = "<$doc_tag_name xmlns:gs2=\"http://www.greenstone.org/gs2\" file=\"$file\" gs2:id=\"$gs2_id\" gs2:docOID=\"$gs2_docOID\">\n";
-    my $documentendtag = "\n</$doc_tag_name>\n";
+    my $documenttag = undef;
+    my $documentendtag = undef;
+
+    $documenttag = "<$doc_tag_name xmlns:gs2=\"http://www.greenstone.org/gs2\" file=\"$file\" gs2:id=\"$gs2_id\" gs2:docOID=\"$gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
+    $documentendtag = "\n</$doc_tag_name>\n";
 
     my $sec_tag_name = "";
@@ -123 +135 @@
     $self->{'num_sections'}++;
 
-    if ($sec_tag_name ne "")
-    {
-        my $sec_gs2_id = $self->{'num_sections'};
-        my $sec_gs2_docOID = $gs2_docOID . "." . $section;
-        $text .= "\n<$sec_tag_name gs2:id=\"$sec_gs2_id\" gs2:docOID=\"$sec_gs2_docOID\">\n";
-    }
+    my $sec_gs2_id = $self->{'num_sections'};
+    my $sec_gs2_docOID = $gs2_docOID;
+    $sec_gs2_docOID .= ".$section" if ($section ne "");
 
     # if we are doing subcollections, then some docs shouldn't be indexed.
@@ -135 +144 @@
     my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
     if (($indexed_doc == 0) || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
-        $text .= "\n</$sec_tag_name>\n" if ($sec_tag_name ne "");
+        if ($sec_tag_name ne "") {
+        $text .= "\n<$sec_tag_name gs2:id=\"$sec_gs2_id\" gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"ignore\">\n";
+        $text .= "\n</$sec_tag_name>\n" 
+        }
             $section = $doc_obj->get_next_section($section);
         next;
           }
 
-    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+    if ($sec_tag_name ne "")
+    {
+        $text .= "\n<$sec_tag_name gs2:id=\"$sec_gs2_id\" gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
+    }
+
+    if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+        $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+    }
+    else {
+        # delete
+        $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
+    }
+
 
     # has the user added a 'metadata' index?
@@ -233 +257 @@
         # filter the text
         $new_text = $self->filter_text ($field, $new_text);
-        $self->{'num_processed_bytes'} += length ($new_text);
+
+        if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+            $self->{'num_processed_bytes'} += length ($new_text);
+            $text .= "$new_text";
+        }
+        else {
+            # delete
+            $self->{'num_processed_bytes'} -= length ($new_text);
+        }
         
-        $text .= "$new_text";
 
         if ($self->{'indexing_text'} && $new_field) {
@@ -287 +318 @@
         $new_text = $self->filter_text ("metadata", $new_text);
         
-        $self->{'num_processed_bytes'} += length ($new_text);
-        $text .= "$new_text";
-
-        
+        if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+        $self->{'num_processed_bytes'} += length ($new_text);
+        $text .= "$new_text";
+        }
+        else {
+        # delete
+        $self->{'num_processed_bytes'} -= length ($new_text);
+        }       
     }
 
@@ -302 +337 @@
         $new_text = $self->filter_text ("allfields", $new_text);
         
-        $self->{'num_processed_bytes'} += length ($new_text);
-        $text .= "$new_text";
-    }
-    
+        if (($edit_mode eq "add") || ($edit_mode eq "reindex")) {
+        $self->{'num_processed_bytes'} += length ($new_text);
+        $text .= "$new_text";
+        }
+        else {
+        # delete
+        $self->{'num_processed_bytes'} -= length ($new_text);
+        }
+    }
+        
     $text .= "\n</$sec_tag_name>\n" if ($sec_tag_name ne "");
 
         $section = $doc_obj->get_next_section($section);
-    } #while defined section
-    print $handle "$text\n$documentendtag";
-    #print STDOUT "$text\n$documentendtag";
-}
+    } # while defined section
+
+    print $lucenehandle "$text\n$documentendtag";
+
+##    if ($edit_mode eq "delete") { 
+##       print STDERR "$text\n$documentendtag";
+##    }
+
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj,$file) = @_;
+
+    $self->textedit($doc_obj,$file,"add");
+}
+
+sub textreindex
+{
+    my $self = shift (@_);
+    my ($doc_obj,$file) = @_;
+
+    $self->textedit($doc_obj,$file,"reindex");
+}
+
+sub textdelete
+{
+    my $self = shift (@_);
+    my ($doc_obj,$file) = @_;
+
+    $self->textedit($doc_obj,$file,"delete");
+}
+
+
+
+
 
 # /** We make this builder pretend to be a document processor so we can get
@@ -494 +567 @@
 1;
 
+