Changeset 6584 for trunk/gsdl/src/recpt/querytools.cpp

Timestamp:

2004-01-22T14:17:30+13:00 (20 years ago)

Author:

kjdon

Message:

Fiddled around with segmenting for chinese text. Haven't changed how the
segmentation is done, or what character ranges are used.
But when its done is now controlled by the collect.cfg. There is a new
option, separate_cjk, values true or false, default false. Segmentation
is only done if this is set to true. This is passed as a global option to
all plugins by the import.pl script, so the user just needs to add it
once to the config file, not as an option to all plugins.
The queryaction uses this option too to determine whether or not to segment
the query.

File:

• trunk/gsdl/src/recpt/querytools.cpp (modified) (3 diffs)

trunk/gsdl/src/recpt/querytools.cpp

@@ -154 +154 @@
 }
 
-void format_querystring (text_t &querystring, int querymode) {
+void format_querystring (text_t &querystring, int querymode, bool segment) {
   text_t formattedstring;
 
+  if (querymode == 1 && !segment) return;
+  
   text_t::const_iterator here = querystring.begin();
   text_t::const_iterator end = querystring.end();
@@ -171 +173 @@
                  *here == '!' || *here == '&')) {
       formattedstring.push_back(' ');
-    } else {
+    } else if (segment) {
       if ((*here >= 0x4e00 && *here <= 0x9fa5) ||
       (*here >= 0xf900 && *here <= 0xfa2d)) {
@@ -184 +186 @@
     space = false;
       }
+    
+    } else {
+      formattedstring.push_back (*here);
     }
     here ++;