Changeset 9186

Timestamp:

2005-02-25T14:27:16+13:00 (19 years ago)

Author:

kjdon

Message:

changed the doc id numbers if no sections to be doc numbers not section numbers, and delete entities from the text and metadata before indexing - undefined entities cause an xml parse exception

File:

• trunk/gsdl/perllib/lucenebuildproc.pm (modified) (7 diffs)

trunk/gsdl/perllib/lucenebuildproc.pm

@@ -32 +32 @@
 
 use mgppbuildproc; 
-
+use ghtml;
 
 sub BEGIN {
@@ -44 +44 @@
 
     return bless $self, $class;
+}
+
+sub preprocess_text {
+    my $self = shift (@_);
+    my ($text, $strip_html, $para) = @_;
+
+    my ($outtext) = "";
+    if ($strip_html) { 
+    while ($text =~ /<([^>]*)>/ && $text ne "") {
+        
+        my $tag = $1;
+        $outtext .= $`." "; #add everything before the matched tag
+        $text = $'; #'everything after the matched tag
+        if ($para && $tag =~ /^\s*p\s/i) {
+        $outtext .= $para;
+        }
+        elsif ($tag =~ /^pre$/) { # a pre tag
+        $text =~ /<\/pre>/; # find the closing pre tag
+        my $tmp_text = $`; #everything before the closing pre tag
+        $text = $'; #'everything after the </pre>
+        $tmp_text =~ s/[<>]//g; # remove all < and >
+        $outtext.= $tmp_text . " ";
+        }
+    }
+    
+    $outtext .= $text; # add any remaining text
+    } #if strip_html
+    else {
+    $outtext = $text;
+    }
+    #if ($para) {
+    #$text =~ s/(<p\b)/$para$1/gi;
+    #return $text;
+    #}
+
+    # remove entities
+    $outtext =~ s/&\w{1,10};//g; 
+
+    return $outtext;
 }
 
@@ -102 +141 @@
     my $docid="";
     if ($ldoc_level) {
-    my $doc_sec_num = $self->{'num_sections'}+1;    
-    $docid = "gs2:id=\"$doc_sec_num\"";
+    if ($lsec_level) {
+        
+        my $doc_sec_num = $self->{'num_sections'}+1;    
+        $docid = "gs2:id=\"$doc_sec_num\"";
+    } else {
+        my $doc_sec_num = $self->{'num_docs'};  
+        $docid = "gs2:id=\"$doc_sec_num\"";
+    }
     }
 
@@ -175 +220 @@
                 if ($self->{'store_text'}) {
                 $tmp_text .= $doc_obj->get_text ($section);
-                $tmp_text =~ s/</&lt;/g;
-                $tmp_text =~ s/>/&gt;/g;
+                &ghtml::htmlsafe($tmp_text);
                 $new_text .= $tmp_text;
                 }
@@ -227 +271 @@
                 foreach $item (@{$doc_obj->get_metadata ($section, $real_field)}) {
                 $new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n";
+                # remove entities
+                $new_text =~ s/&\w{1,10};//g; 
                 }
             }
@@ -234 +280 @@
             # filter the text
             $self->filter_text ($field, $new_text);
-
             $self->{'num_processed_bytes'} += length ($new_text);
             $text .= "$new_text";
@@ -246 +291 @@
     } #while defined section
     print $handle "$text\n$documentendtag"; 
-    
 }