Changeset 17110 for gsdl/trunk

Timestamp:

2008-08-30T07:35:20+12:00 (16 years ago)

Author:

kjdon

Message:

changed way cjk separation is done. Not done in plugins any more, but is now an indexoption. cnseg called from filter_text method. generate_index_options sets up the field in buildproc

Location:

gsdl/trunk/perllib

Files:

• basebuilder.pm (modified) (4 diffs)
• basebuildproc.pm (modified) (6 diffs)
• lucenebuilder.pm (modified) (1 diff)
• lucenebuildproc.pm (modified) (1 diff)
• mgbuilder.pm (modified) (2 diffs)
• mgbuildproc.pm (modified) (2 diffs)
• mgppbuilder.pm (modified) (1 diff)
• mgppbuildproc.pm (modified) (3 diffs)

gsdl/trunk/perllib/basebuilder.pm

@@ -148 +148 @@
     
     $self->generate_index_list();
-    $self->generate_index_options();
-
+ 
     # sort out subcollection indexes
     if (defined $self->{'collect_cfg'}->{'indexsubcollections'}) {
@@ -216 +215 @@
     die "$@" if $@;
 
+    
+   $self->generate_index_options();
+
     if (!$self->{'debug'} && !$self->{'keepold'}) {
     # remove any old builds
@@ -234 +236 @@
 }
 
+sub generate_index_options {
+    my $self = shift (@_);
+
+    my $separate_cjk = 0;
+    
+    if (defined($self->{'collect_cfg'}->{'indexoptions'})) {
+    foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {
+        if ($option =~ /separate_cjk/) {
+        $separate_cjk = 1;
+        }
+    }
+    }
+    # set this for building
+    $self->{'buildproc'}->set_separate_cjk($separate_cjk);
+    # record it for build.cfg
+    $self->{'separate_cjk'} = $separate_cjk;
+}
+ 
 sub set_sections_index_document_metadata {
     my $self = shift (@_);
@@ -429 +449 @@
     $build_cfg->{'indexstem'} = &util::get_dirsep_tail($self->{'collection'});
     $build_cfg->{'stemindexes'} = $self->{'stemindexes'};
+    if ($self->{'separate_cjk'}) {
+    $build_cfg->{'separate_cjk'} = "true";
+    }
     
     # store the number of documents and number of bytes

gsdl/trunk/perllib/basebuildproc.pm

@@ -68 +68 @@
     $self->{'indexexparr'} = [];
 
+    $self->{'separate_cjk'} = 0;
+
     my $found_num_data = 0;
     my $buildconfigfile = undef;
@@ -74 +76 @@
     # For incremental building need to seed num_docs etc from values
     # stored in build.cfg (if present)
-      print STDERR "Keepold!\n";
     $buildconfigfile = &util::filename_cat($build_dir, "build.cfg");
-      print STDERR "Build cfg: $buildconfigfile\n";
     if (-e $buildconfigfile) {
         $found_num_data = 1;
@@ -84 +84 @@
         $buildconfigfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
                            "index", "build.cfg");
-            print STDERR "Index cfg: $buildconfigfile\n";
         if (-e $buildconfigfile) {
         $found_num_data = 1;
@@ -91 +90 @@
 
     }
-    #else
-    #  {
-    #    print STDERR "Removeold!\n";
-    #  }
 
     if ($found_num_data)
@@ -308 +303 @@
     $self->{'sections_index_document_metadata'} = $index_type;
 }
+
+sub set_separate_cjk {
+    my $self = shift (@_);
+    my ($sep_cjk) = @_;
+
+    $self->{'separate_cjk'} = $sep_cjk;
+}
+
 sub process {
     my $self = shift (@_);
@@ -315 +318 @@
 }
 
+# post process text depending on field. Currently don't do anything here
+# except cjk separation
+sub filter_text {
+    my $self = shift (@_);
+    my ($field, $text) = @_;
+
+    # lets do cjk seg here
+    my $new_text =$text;
+    if ($self->{'separate_cjk'}) {
+    $new_text = &cnseg::segment($text);
+    }
+    return $new_text;
+}
 
 

gsdl/trunk/perllib/lucenebuilder.pm

@@ -85 +85 @@
     my $self = shift (@_);
 
+    $self->SUPER::generate_index_options();
+    
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;

gsdl/trunk/perllib/lucenebuildproc.pm

@@ -270 +270 @@
         }
         # filter the text
-        $self->filter_text ($field, $new_text);
+        $new_text = $self->filter_text ($field, $new_text);
         $self->{'num_processed_bytes'} += length ($new_text);
 

gsdl/trunk/perllib/mgbuilder.pm

@@ -82 +82 @@
 sub generate_index_options {
     my $self = shift (@_);
-
+    $self->SUPER::generate_index_options();
+    
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;
@@ -91 +92 @@
     $self->{'casefold'} = 1;
     $self->{'stem'} = 1;
+    
     } else {
     foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {

gsdl/trunk/perllib/mgbuildproc.pm

@@ -47 +47 @@
 sub find_paragraphs {
     $_[1] =~ s/(<p\b)/\cC$1/gi;
-}
-
-sub filter_text {
-    # $self->filter_text ($field, $new_text);
-    # don't want to do anything for this version, however,
-    # in a particular collection you might want to override
-    # this method to post-process certain fields depending on
-    # the field, or whether we are outputting it for indexing
 }
 
@@ -131 +123 @@
             
             # filter the text
-            $self->filter_text ($field, $new_text);
+            $new_text = $self->filter_text ($field, $new_text);
 
             $text .= "$new_text\cC";

gsdl/trunk/perllib/mgppbuilder.pm

@@ -140 +140 @@
     my $self = shift (@_);
 
+    $self->SUPER::generate_index_options();
+
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;

gsdl/trunk/perllib/mgppbuildproc.pm

@@ -31 +31 @@
 
 use basebuildproc;
+use cnseg;
+
 use strict;
 no strict 'refs'; # allow filehandles to be variables and viceversa
@@ -167 +169 @@
 }
     
-    
-
-sub filter_text {
-    # $self->filter_text ($field, $new_text);
-    # don't want to do anything for this version, however,
-    # in a particular collection you might want to override
-    # this method to post-process certain fields depending on
-    # the field, or whether we are outputting it for indexing
-}
-
 sub text {
     my $self = shift (@_);
@@ -326 +318 @@
         
         # filter the text
-        $self->filter_text ($field, $new_text);
+        $new_text = $self->filter_text ($field, $new_text);
         
         $self->{'num_processed_bytes'} += length ($new_text);