Index: /gsdl/trunk/perllib/basebuilder.pm
===================================================================
--- /gsdl/trunk/perllib/basebuilder.pm	(revision 17109)
+++ /gsdl/trunk/perllib/basebuilder.pm	(revision 17110)
@@ -148,6 +148,5 @@
     
     $self->generate_index_list();
-    $self->generate_index_options();
-
+ 
     # sort out subcollection indexes
     if (defined $self->{'collect_cfg'}->{'indexsubcollections'}) {
@@ -216,4 +215,7 @@
     die "$@" if $@;
 
+    
+   $self->generate_index_options();
+
     if (!$self->{'debug'} && !$self->{'keepold'}) {
 	# remove any old builds
@@ -234,4 +236,22 @@
 }
 
+sub generate_index_options {
+    my $self = shift (@_);
+
+    my $separate_cjk = 0;
+    
+    if (defined($self->{'collect_cfg'}->{'indexoptions'})) {
+	foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {
+	    if ($option =~ /separate_cjk/) {
+		$separate_cjk = 1;
+	    }
+	}
+    }
+    # set this for building
+    $self->{'buildproc'}->set_separate_cjk($separate_cjk);
+    # record it for build.cfg
+    $self->{'separate_cjk'} = $separate_cjk;
+}
+ 
 sub set_sections_index_document_metadata {
     my $self = shift (@_);
@@ -429,4 +449,7 @@
     $build_cfg->{'indexstem'} = &util::get_dirsep_tail($self->{'collection'});
     $build_cfg->{'stemindexes'} = $self->{'stemindexes'};
+    if ($self->{'separate_cjk'}) {
+	$build_cfg->{'separate_cjk'} = "true";
+    }
     
     # store the number of documents and number of bytes
Index: /gsdl/trunk/perllib/basebuildproc.pm
===================================================================
--- /gsdl/trunk/perllib/basebuildproc.pm	(revision 17109)
+++ /gsdl/trunk/perllib/basebuildproc.pm	(revision 17110)
@@ -68,4 +68,6 @@
     $self->{'indexexparr'} = [];
 
+    $self->{'separate_cjk'} = 0;
+
     my $found_num_data = 0;
     my $buildconfigfile = undef;
@@ -74,7 +76,5 @@
 	# For incremental building need to seed num_docs etc from values
 	# stored in build.cfg (if present)
-      print STDERR "Keepold!\n";
 	$buildconfigfile = &util::filename_cat($build_dir, "build.cfg");
-      print STDERR "Build cfg: $buildconfigfile\n";
 	if (-e $buildconfigfile) {
 	    $found_num_data = 1;
@@ -84,5 +84,4 @@
 	    $buildconfigfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
 						   "index", "build.cfg");
-            print STDERR "Index cfg: $buildconfigfile\n";
 	    if (-e $buildconfigfile) {
 		$found_num_data = 1;
@@ -91,8 +90,4 @@
 
     }
-    #else
-    #  {
-    #    print STDERR "Removeold!\n";
-    #  }
 
     if ($found_num_data)
@@ -308,4 +303,12 @@
     $self->{'sections_index_document_metadata'} = $index_type;
 }
+
+sub set_separate_cjk {
+    my $self = shift (@_);
+    my ($sep_cjk) = @_;
+
+    $self->{'separate_cjk'} = $sep_cjk;
+}
+
 sub process {
     my $self = shift (@_);
@@ -315,4 +318,17 @@
 }
 
+# post process text depending on field. Currently don't do anything here
+# except cjk separation
+sub filter_text {
+    my $self = shift (@_);
+    my ($field, $text) = @_;
+
+    # lets do cjk seg here
+    my $new_text =$text;
+    if ($self->{'separate_cjk'}) {
+	$new_text = &cnseg::segment($text);
+    }
+    return $new_text;
+}
 
 
Index: /gsdl/trunk/perllib/lucenebuilder.pm
===================================================================
--- /gsdl/trunk/perllib/lucenebuilder.pm	(revision 17109)
+++ /gsdl/trunk/perllib/lucenebuilder.pm	(revision 17110)
@@ -85,4 +85,6 @@
     my $self = shift (@_);
 
+    $self->SUPER::generate_index_options();
+    
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;
Index: /gsdl/trunk/perllib/lucenebuildproc.pm
===================================================================
--- /gsdl/trunk/perllib/lucenebuildproc.pm	(revision 17109)
+++ /gsdl/trunk/perllib/lucenebuildproc.pm	(revision 17110)
@@ -270,5 +270,5 @@
 	    }
 	    # filter the text
-	    $self->filter_text ($field, $new_text);
+	    $new_text = $self->filter_text ($field, $new_text);
 	    $self->{'num_processed_bytes'} += length ($new_text);
 
Index: /gsdl/trunk/perllib/mgbuilder.pm
===================================================================
--- /gsdl/trunk/perllib/mgbuilder.pm	(revision 17109)
+++ /gsdl/trunk/perllib/mgbuilder.pm	(revision 17110)
@@ -82,5 +82,6 @@
 sub generate_index_options {
     my $self = shift (@_);
-
+    $self->SUPER::generate_index_options();
+    
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;
@@ -91,4 +92,5 @@
 	$self->{'casefold'} = 1;
 	$self->{'stem'} = 1;
+	
     } else {
 	foreach my $option (@{$self->{'collect_cfg'}->{'indexoptions'}}) {
Index: /gsdl/trunk/perllib/mgbuildproc.pm
===================================================================
--- /gsdl/trunk/perllib/mgbuildproc.pm	(revision 17109)
+++ /gsdl/trunk/perllib/mgbuildproc.pm	(revision 17110)
@@ -47,12 +47,4 @@
 sub find_paragraphs {
     $_[1] =~ s/(<p\b)/\cC$1/gi;
-}
-
-sub filter_text {
-    # $self->filter_text ($field, $new_text);
-    # don't want to do anything for this version, however,
-    # in a particular collection you might want to override
-    # this method to post-process certain fields depending on
-    # the field, or whether we are outputting it for indexing
 }
 
@@ -131,5 +123,5 @@
 		    
 		    # filter the text
-		    $self->filter_text ($field, $new_text);
+		    $new_text = $self->filter_text ($field, $new_text);
 
 		    $text .= "$new_text\cC";
Index: /gsdl/trunk/perllib/mgppbuilder.pm
===================================================================
--- /gsdl/trunk/perllib/mgppbuilder.pm	(revision 17109)
+++ /gsdl/trunk/perllib/mgppbuilder.pm	(revision 17110)
@@ -140,4 +140,6 @@
     my $self = shift (@_);
 
+    $self->SUPER::generate_index_options();
+
     $self->{'casefold'} = 0;
     $self->{'stem'} = 0;
Index: /gsdl/trunk/perllib/mgppbuildproc.pm
===================================================================
--- /gsdl/trunk/perllib/mgppbuildproc.pm	(revision 17109)
+++ /gsdl/trunk/perllib/mgppbuildproc.pm	(revision 17110)
@@ -31,4 +31,6 @@
 
 use basebuildproc;
+use cnseg;
+
 use strict;
 no strict 'refs'; # allow filehandles to be variables and viceversa
@@ -167,14 +169,4 @@
 }
 	
-    
-
-sub filter_text {
-    # $self->filter_text ($field, $new_text);
-    # don't want to do anything for this version, however,
-    # in a particular collection you might want to override
-    # this method to post-process certain fields depending on
-    # the field, or whether we are outputting it for indexing
-}
-
 sub text {
     my $self = shift (@_);
@@ -326,5 +318,5 @@
 	    
 	    # filter the text
-	    $self->filter_text ($field, $new_text);
+	    $new_text = $self->filter_text ($field, $new_text);
 	    
 	    $self->{'num_processed_bytes'} += length ($new_text);
