Changeset 9179

Timestamp:

2005-02-24T16:57:53+13:00 (19 years ago)

Author:

kjdon

Message:

added a text pass to store the documents - we want all the html in it (but escaped) and no metadata

File:

• trunk/gsdl/perllib/lucenebuilder.pm (modified) (3 diffs)

trunk/gsdl/perllib/lucenebuilder.pm

@@ -66 +66 @@
 }
 
+# this writes a nice version of the text docs
 sub compress_text {
 
     my $self = shift (@_);
     my ($textindex) = @_;
+    my $outhandle = $self->{'outhandle'};
+    print STDERR "Saving the document text\n";
+    # the text directory
+    my $text_dir = &util::filename_cat($self->{'build_dir'}, "text");
+    my $build_dir = $self->{'build_dir'};
+    &util::mk_all_dir ($text_dir);
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+    $text_dir =~ s@/@\\@g;
+    } else {
+    if ($outhandle ne "STDERR") {
+        # so lucene_passes doesn't print to stderr if we redirect output
+        $osextra .= " 2>/dev/null";
+    }
+    }
+    
+
+    # get any os specific stuff
+    my $scriptdir = "$ENV{'GSDLHOME'}/bin/script";
+
+    my $lucene_passes_exe = &util::filename_cat($scriptdir, "lucene_passes.pl");
+    my $lucene_passes_sections = "Doc";
+
+    my ($handle);
+
+    if ($self->{'debug'}) {
+    $handle = STDOUT;
+    } else {
+    if (!-e "$lucene_passes_exe" || 
+        !open (PIPEOUT, "| $lucene_passes_exe text $lucene_passes_sections \"$build_dir\" \"poo\"   $osextra")) {
+        print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
+        die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";
+    }
+    $handle = lucenebuilder::PIPEOUT;
+    }
+    my $levels = $self->{'levels'};
+    undef $levels->{'paragraph'}; # get rid of para if we had it.
+    # set up the document processr
+    $self->{'buildproc'}->set_output_handle ($handle);
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($textindex);
+    $self->{'buildproc'}->set_indexing_text (0);
+    $self->{'buildproc'}->set_store_text(1);
+    $self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
+    $self->{'buildproc'}->set_levels ($levels);                       
+    $self->{'buildproc'}->reset();
+    &plugin::begin($self->{'pluginfo'}, $self->{'source_dir'}, 
+           $self->{'buildproc'}, $self->{'maxdocs'});    
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+           "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+    &plugin::end($self->{'pluginfo'});
+    close ($handle) unless $self->{'debug'};
+    close PIPEOUT;
+    $self->print_stats();
+
+    print STDERR "</Stage>\n" if $self->{'gli'};
+
 }
 
@@ -94 +153 @@
         my $idx = $self->{'index_mapping'}->{$index};
         foreach my $level (keys %{$self->{'levels'}}) {
+        next if $level =~ /paragraph/; # we don't do para indexing
         my ($pindex) = $level =~ /^(.)/;
         # should probably check that new name with level 
@@ -199 +259 @@
     } else {
     if (!-e "$lucene_passes_exe" || 
-        !open (PIPEOUT, "| $lucene_passes_exe $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
+        !open (PIPEOUT, "| $lucene_passes_exe index $lucene_passes_sections \"$build_dir\" \"$indexdir\"   $osextra")) {
         print STDERR "<FatalError name='NoRunLucenePasses'/>\n</Stage>\n" if $self->{'gli'};
         die "lucenebuilder::build_index - couldn't run $lucene_passes_exe\n";