Changeset 14068

Timestamp:

2007-05-14T11:34:42+12:00 (17 years ago)

Author:

mdewsnip

Message:

A new preprocess_text function that is similar to the mgppbuildproc version but adds extra spaces to prevent problems with words joining together when the XML tags are removed by Lucene. Many thanks to John Thompson from DL Consulting Ltd (ttp://www.dlconsulting.com).

File:

• trunk/gsdl/perllib/lucenebuildproc.pm (modified) (1 diff)

trunk/gsdl/perllib/lucenebuildproc.pm

@@ -418 +418 @@
 }
 
+
+# This is similar to mgppbuildproc's preprocess_text but adds extra spaces
+# Otherwise the removal of tags below might lead to Lucene turning
+#   "...farming</p>\n<p>EDWARD.." into "farmingedward"
+#     (example from demo collection b20cre)
+# Many thanks to John Thompson, DL Consulting Ltd. (www.dlconsulting.com)
+sub preprocess_text
+{
+    my $self = shift (@_);
+    my ($text, $strip_html, $para) = @_;
+    # at this stage, we do not do paragraph tags unless have strip_html -
+    # it will result in a huge mess of non-xml
+    return unless $strip_html;
+
+    my $new_text = $text;
+
+    # if we have <pre> tags, we can have < > inside them, need to delete
+    # the <> before stripping tags
+    $new_text =~ s/<pre>(.*?)<\/pre>/$self->remove_gtlt($1,$para)/gse;
+
+    if ($para eq "") {
+       # just remove all tags
+       $new_text =~ s/<[^>]*>/ /gs;
+    } else {
+       # strip all tags except <p> tags which get turned into $para
+       $new_text =~ s/<([^>]*)>/$self->process_tags($1, $para)/gse;
+    }
+
+    return $new_text;
+}
+
+
 1;