Changeset 4769

Timestamp:

2003-06-23T15:01:38+12:00 (21 years ago)

Author:

kjdon

Message:

paragraph indexing should work now, also slightly changed some other internal stuff to do with remembering what fields were found when indexing 'metadata'

File:

• trunk/gsdl/perllib/mgppbuildproc.pm (modified) (12 diffs)

trunk/gsdl/perllib/mgppbuildproc.pm

@@ -79 +79 @@
     $self->{'dontindex'} = {};
     $self->{'indexfieldmap'} = {};
-    $self->{'indexfields'} = {};
+    $self->{'indexfields'} = {}; # only put in the ones that are not specified directly in the index
     $self->{'strip_html'}=1;
 
@@ -444 +444 @@
 }
 
-sub find_paragraphs {
-    $_[1] =~ s/(<p\b)/<Paragraph>$1/gi;
-}
+#sub find_paragraphs {
+#    $_[1] =~ s/(<p\b)/<Paragraph>$1/gi;
+#}
 
 #this function strips the html tags from the doc if ($strip_html) and
@@ -456 +456 @@
     my $self = shift (@_);
     my ($text, $strip_html, $para) = @_;
-
     my ($outtext) = "";
     if ($strip_html) { 
@@ -464 +463 @@
         $outtext .= $`." "; #add everything before the matched tag
         $text = $'; #everything after the matched tag
-        if ($para && $tag =~ /^\s*p\s/) {
-        $outtext .= "<Paragraph> ";
+        if ($para && $tag =~ /^\s*p\s/i) {
+        $outtext .= $para;
         }
         elsif ($tag =~ /^pre$/) { # a pre tag
@@ -480 +479 @@
     } #if strip_html
 
-    if ($para) {
-    $text =~ s/(<p\b)/<Paragraph>$1/gi;
-    return $text;
-    }
+    #if ($para) {
+    #$text =~ s/(<p\b)/$para$1/gi;
+    #return $text;
+   # }
     return $text;
 }
@@ -501 +500 @@
     my ($doc_obj) = @_;
     my $handle = $self->{'output_handle'};
+    my $outhandle = $self->{'outhandle'};
     my $indexed_doc = 1;
 
@@ -541 +541 @@
     my ($fields) = $self->{'index'};
 
+    my ($documenttag) = "";
+    my($documentendtag) = "";
+    #if ($self->{'levels'}->{'Document'}) {
+    $documenttag = "\n<Document>\n";
+    $documentendtag = "</Document>\n";
+    #}
     my ($sectiontag) = "";
     if ($self->{'levels'}->{'Section'}) {
@@ -547 +553 @@
     my ($paratag) = "";
     if ($self->{'levels'}->{'Paragraph'}) {
-    $paratag = "<Paragraph>";
+    if ($self->{'strip_html'}) {
+        $paratag = "<Paragraph>";
+    } else {
+        print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
+    }
     }
     my $doc_section = 0; # just for this document
-    my $text = "<Document>\n";
+    
+    my $text = $documenttag;
    
     # get the text for this document
@@ -558 +569 @@
     $doc_section++;
     $self->{'num_sections'} += 1;
-    $text .= $sectiontag;
+    $text .= "$sectiontag";
 
     if ($indexed_doc) {
+        if ($self->{'indexing_text'}) {
+        $text .= "$paratag"; # only add para tags for indexing
+        # note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text
+        }
         $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
         foreach my $field (split (/,/, $fields)) {
@@ -571 +586 @@
             if ($real_field eq "text") {
             if ($self->{'indexing_text'}) { #tag the text with <Text>...</Text>, add the <Paragraph> tags and strip out html if needed
-                $new_text .= "<TX>\n";
-                $tmp_text .= $doc_obj->get_text ($section) if $self->{'store_text'};
-                $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, $self->{'levels'}->{'Paragraph'});
-
+                $new_text .= "$paratag<TX>\n";
+                $tmp_text .= $doc_obj->get_text ($section);
+                $tmp_text = $self->preprocess_text($tmp_text, $self->{'strip_html'}, "</TX>$paratag<TX>");
+                
                 $new_text .= "$tmp_text</TX>\n";
-                if (!defined $self->{'indexfields'}->{'TextOnly'}) {
-                $self->{'indexfields'}->{'TextOnly'} = 1;   
-                }
+                #if (!defined $self->{'indexfields'}->{'TextOnly'}) {
+                #$self->{'indexfields'}->{'TextOnly'} = 1;   
+                #}
             }
             else { # leave html stuff in, and dont add Paragraph tags - never retrieve paras at the moment
                 $new_text .= $doc_obj->get_text ($section) if $self->{'store_text'};
-                            #if ($self->{'levels'}->{'Paragraph'}) {
-                #$self->find_paragraphs($new_text);
-                #}                
             }
             } else { # metadata field
-            if ($real_field eq "metadata") { # insert all metadata 
+            if ($real_field eq "allfields") { #ignore
+            }
+            elsif ($real_field eq "metadata") { # insert all metadata 
                 #except gsdl stuff
                 my $shortname = "";
@@ -617 +631 @@
             else { #individual metadata specified
                 my $shortname="";
-                if (!defined $self->{'indexfields'}->{$real_field}) {
-                $self->{'indexfields'}->{$real_field} = 1;
-                }
+                #if (!defined $self->{'indexfields'}->{$real_field}) {
+                #$self->{'indexfields'}->{$real_field} = 1;
+                #}
                 if (defined $self->{'indexfieldmap'}->{$real_field}) {
                 $shortname = $self->{'indexfieldmap'}->{$real_field};
@@ -646 +660 @@
     $section = $doc_obj->get_next_section($section);
     } #while defined section
-    print $handle "$text\n</Document>\n"; 
+    print $handle "$text\n$documentendtag"; 
+    
 }