Changeset 6584

Timestamp:

2004-01-22T14:17:30+13:00 (20 years ago)

Author:

kjdon

Message:

Fiddled around with segmenting for chinese text. Haven't changed how the
segmentation is done, or what character ranges are used.
But when its done is now controlled by the collect.cfg. There is a new
option, separate_cjk, values true or false, default false. Segmentation
is only done if this is set to true. This is passed as a global option to
all plugins by the import.pl script, so the user just needs to add it
once to the config file, not as an option to all plugins.
The queryaction uses this option too to determine whether or not to segment
the query.

Location:

trunk/gsdl

Files:

• bin/script/import.pl (modified) (3 diffs)
• perllib/colcfg.pm (modified) (3 diffs)
• perllib/mgbuilder.pm (modified) (1 diff)
• perllib/mgppbuilder.pm (modified) (2 diffs)
• perllib/plugin.pm (modified) (3 diffs)
• perllib/plugins/BasPlug.pm (modified) (2 diffs)
• src/colservr/collectserver.cpp (modified) (1 diff)
• src/recpt/comtypes.cpp (modified) (1 diff)
• src/recpt/comtypes.h (modified) (1 diff)
• src/recpt/documentaction.cpp (modified) (1 diff)
• src/recpt/queryaction.cpp (modified) (6 diffs)
• src/recpt/queryaction.h (modified) (1 diff)
• src/recpt/querytools.cpp (modified) (3 diffs)
• src/recpt/querytools.h (modified) (1 diff)

trunk/gsdl/bin/script/import.pl

@@ -348 +348 @@
     # options must be known before we read the collect.cfg))
     my $plugins = [];
+    my @global_opts = ();
+    
     $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
     if (-e $configfilename) {
@@ -414 +416 @@
         $gli = 1;
     }
+
+    # global plugin stuff
+    if (defined $collectcfg->{'separate_cjk'}&& $collectcfg->{'separate_cjk'} =~ /^true$/i) {
+        push @global_opts, "-separate_cjk";
+    }
+    
 
     } else {
@@ -433 +441 @@
 
     # load all the plugins
-    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog);
+    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts);
     if (scalar(@$pluginfo) == 0) {
     print $out &lookup_string("{import.no_plugins_loaded}") . "\n";

trunk/gsdl/perllib/colcfg.pm

@@ -44 +44 @@
 # {'buildtype'}->string
 # {'maxnumeric'}->string
+# {'separate_cjk'}->string
 
 # {'maintainer'}->array of strings
@@ -76 +77 @@
                     q/verbosity|allclassifications|OIDtype|maxdocs|/ .
                     q/groupsize|sortmeta|debug|mode|create_images|/ .
-                    q/maxnumeric)$/,
+                    q/maxnumeric|separate_cjk)$/,
                     q/(maintainer|languages|indexsubcollections|/ .
                        q/indexes|dontbuild|dontgdbm|mirror|phind|levels|searchtype)$/,
@@ -91 +92 @@
                  q/archivedir|cachedir|builddir|removeold|/ .
                  q/textcompress|buildtype|collectdir|no_text|/ .
-                 q/allclassifications|maxnumeric)$/,
+                 q/allclassifications|maxnumeric|separate_cjk)$/,
                  q/^(maintainer|languages|indexsubcollections|/ .
                  q/indexes|dontbuild|dontgdbm|levels|searchtype)$/,

trunk/gsdl/perllib/mgbuilder.pm

@@ -151 +151 @@
     
     # load all the plugins
-    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle, $failhandle);
+
+    #build up the extra global options for the plugins
+    my @global_opts = ();
+    if (defined $self->{'collect_cfg'}->{'separate_cjk'} && $self->{'collect_cfg'}->{'separate_cjk'} =~ /^true$/i) {
+    push @global_opts, "-separate_cjk";
+    }
+    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle, $failhandle, \@global_opts);
     if (scalar(@{$self->{'pluginfo'}}) == 0) {
     print $outhandle "No plugins were loaded.\n";

trunk/gsdl/perllib/mgppbuilder.pm

@@ -211 +211 @@
     print $outhandle "doclevel = ". $self->{'doc_level'}."\n";
     # get the list of plugins for this collection
+
+    #build up the extra global options for the plugins
+    my @global_opts = ();
+    if (defined $self->{'collect_cfg'}->{'separate_cjk'} && $self->{'collect_cfg'}->{'separate_cjk'} =~ /^true$/i) {
+    push @global_opts, "-separate_cjk";
+    }
+
     my $plugins = [];
     if (defined $self->{'collect_cfg'}->{'plugin'}) {
@@ -217 +224 @@
     
     # load all the plugins
-    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle);
+    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle, \@global_opts);
     if (scalar(@{$self->{'pluginfo'}}) == 0) {
     print $outhandle "No plugins were loaded.\n";

trunk/gsdl/perllib/plugin.pm

@@ -41 +41 @@
 }
 
-
+#globaloptions contains any options that should be passed to all plugins
 sub load_plugins {
     my ($plugin_list) = shift @_;
-    ($verbosity, $outhandle, $failhandle) = @_; # globals
+    ($verbosity, $outhandle, $failhandle, $globaloptions) = @_; # globals
     my @plugin_objects = ();
 
@@ -50 +50 @@
     $outhandle = STDERR unless defined $outhandle;
     $failhandle = STDERR unless defined $failhandle;
+
+    map { $_ = "\"$_\""; } @$globaloptions;
+    my $globals = join (",", @$globaloptions);
 
     foreach $pluginoptions (@$plugin_list) {
@@ -70 +73 @@
     map { $_ = "\"$_\""; } @$pluginoptions;
     my $options = join (",", @$pluginoptions);
+    if ($globals) {
+        if (@$pluginoptions) {
+        $options .= ",";
+        }
+        $options .= "$globals";
+    }
     $options =~ s/\$/\\\$/g;
+    
     eval ("\$plugobj = new \$pluginname($options)");
     die "$@" if $@;

trunk/gsdl/perllib/plugins/BasPlug.pm

@@ -405 +405 @@
              qq^maximum_century/-?\\d{1,2}( ?B\\.C\\.E\\.)?/-1^, \$self->{'max_century'},
              q^cover_image^, \$self->{'cover_image'},
+             q^separate_cjk^, \$self->{'separate_cjk'},
              "allow_extra_options")) {
 
@@ -615 +616 @@
     $reader->read_file ($textref);
 
-    if ($language eq "zh") {
+    #Now segments chinese if the separate_cjk option is set
+    if ($self->{'separate_cjk'}) {
         # segment the Chinese words
         $$textref = &cnseg::segment($$textref);

trunk/gsdl/src/colservr/collectserver.cpp

@@ -101 +101 @@
       collectinfo.searchTypes = cfgline;
     }
-
+    else if (key == "separate_cjk") {
+      if (value == "true") collectinfo.isSegmented = true;
+      else collectinfo.isSegmented = false;
+    }
     // What have we set in our collect.cfg file :  document or collection ?
     else if (key == "authenticate") collectinfo.authenticate = value;

trunk/gsdl/src/recpt/comtypes.cpp

@@ -53 +53 @@
   isPublic=true;
   isBeta=false;
+  isSegmented=false;
   languages.erase(languages.begin(), languages.end());
   ccsCols.erase(ccsCols.begin(), ccsCols.end());

trunk/gsdl/src/recpt/comtypes.h

@@ -90 +90 @@
    bool isPublic;
    bool isBeta;
+  bool isSegmented;
    unsigned long buildDate;
    text_tarray ccsCols;    // empty if collection does not use cross-collection searching

trunk/gsdl/src/recpt/documentaction.cpp

@@ -1008 +1008 @@
   
   if (!args["q"].empty() && args.getintarg("hl")) {
+
+    ColInfoResponse_t *cinfo = recpt->get_collectinfo_ptr (collectproto, collection, logout);
+    bool segment = false;
+    if (cinfo != NULL) {
+      segment = cinfo->isSegmented;
+    }
     FilterRequest_t request;
     comerror_t err;
     request.filterResultOptions = FRmatchTerms;
     text_t formattedstring = args["q"];
-    format_querystring (formattedstring, args.getintarg("b"));
+    format_querystring (formattedstring, args.getintarg("b"), segment);
     set_queryfilter_options (request, formattedstring, args);
     collectproto->filter (args["c"], request, queryresponse, err, logout);

trunk/gsdl/src/recpt/queryaction.cpp

@@ -632 +632 @@
 
 // sets the selection box macros _hselection_, _jselection_, _nselection_ _gselection_, fqfselection_ 
-void queryaction::set_option_macro (const text_t &macroname, text_t current_value, bool display_single, bool add_js_update,
-                    const FilterOption_t &option, displayclass &disp) {
+void queryaction::set_option_macro (const text_t &macroname, 
+                    text_t current_value, 
+                    bool display_single, 
+                    bool add_js_update,
+                    const FilterOption_t &option, 
+                    displayclass &disp) {
   
   if (option.validValues.empty()) return;
@@ -1082 +1086 @@
   isapprox isApprox = Exact;
 
+  // what to do about segmentation for multiple colls??
+  bool segment = false;
   text_t formattedstring = "";
-  get_formatted_query_string(formattedstring, args, disp, logout);
+  get_formatted_query_string(formattedstring, segment, args, disp, logout);
 
   if (formattedstring.empty()) {
@@ -1288 +1294 @@
   }
     
+  bool segment = cinfo->isSegmented;
   browserclass *bptr = browsers->getbrowser (browsertype);
 
@@ -1307 +1314 @@
   request.filterResultOptions = FROID | FRmetadata | FRtermFreq;
   text_t formattedstring = "";
-  get_formatted_query_string(formattedstring, args, disp, logout);
+  get_formatted_query_string(formattedstring, segment, args, disp, logout);
 
 
@@ -1354 +1361 @@
 // also adds dates if appropriate in text search
 void queryaction::get_formatted_query_string (text_t &formattedstring, 
+                          bool segment,
                           cgiargsclass &args, 
                           displayclass &disp,
@@ -1360 +1368 @@
     formattedstring = args["q"];
     // remove & | ! for simple search, insert spaces for chinese
-    format_querystring (formattedstring, args.getintarg("b"));
+    format_querystring (formattedstring, args.getintarg("b"), segment);
     if (args["ct"]=="1") { // mgpp - we need to add in the field info
       format_field_info(formattedstring, args["fqf"]);

trunk/gsdl/src/recpt/queryaction.h

@@ -73 +73 @@
                 int numDocs, isapprox isApprox); 
 
-  void get_formatted_query_string (text_t &formattedstring, cgiargsclass &args, 
+  void get_formatted_query_string (text_t &formattedstring, bool segment, 
+                   cgiargsclass &args, 
                    displayclass &disp, ostream &logout);
   void define_query_interface(displayclass &disp, cgiargsclass &args, 

trunk/gsdl/src/recpt/querytools.cpp

@@ -154 +154 @@
 }
 
-void format_querystring (text_t &querystring, int querymode) {
+void format_querystring (text_t &querystring, int querymode, bool segment) {
   text_t formattedstring;
 
+  if (querymode == 1 && !segment) return;
+  
   text_t::const_iterator here = querystring.begin();
   text_t::const_iterator end = querystring.end();
@@ -171 +173 @@
                  *here == '!' || *here == '&')) {
       formattedstring.push_back(' ');
-    } else {
+    } else if (segment) {
       if ((*here >= 0x4e00 && *here <= 0x9fa5) ||
       (*here >= 0xf900 && *here <= 0xfa2d)) {
@@ -184 +186 @@
     space = false;
       }
+    
+    } else {
+      formattedstring.push_back (*here);
     }
     here ++;

trunk/gsdl/src/recpt/querytools.h

@@ -40 +40 @@
 void set_more_queryfilter_options (FilterRequest_t &request, cgiargsclass &args);
 
-void format_querystring (text_t &querystring, int querymode);
+void format_querystring (text_t &querystring, int querymode, bool segment);
 
 void add_dates(text_t &querystring, int startdate, int enddate,