Changeset 16980

Timestamp:

2008-08-25T09:58:13+12:00 (16 years ago)

Author:

kjdon

Message:

cjk character segmentation. text_t chars not big enough to handle numbers > 0xffff. have commented these ranges out in c++ and perl until we implement a better solution. these high ranges are only for extension sets anyway, so most common words will be segmented

Location:

gsdl/trunk

Files:

• perllib/cnseg.pm (modified) (1 diff)
• runtime-src/src/recpt/querytools.cpp (modified) (1 diff)

gsdl/trunk/perllib/cnseg.pm

@@ -55 +55 @@
     my $space = 1; # start doesn't need a space
     foreach $c (@$uniin) {
-    if (($c >= 0x2e80 && $c <= 0xfa6a) || # main east asian codes
-        ($c >= 0x20000 && $c <= 0x2a6d6) || # cjk unified ideographs ext B
-        ($c >= 0x2f800 && $c <= 0x2fa1d)) { #cjk compatibility ideographs supplement
+    if (($c >= 0x2e80 && $c <= 0xd7a3) ||
+        ( $c >= 0xf900 && $c <= 0xfa6a)) { # main east asian codes
+        # currently c++ receptionist code can't handle these large numbers
+        # search terms need to be segmented the same way. Add these back
+        # in when fix up c++
+       # ($c >= 0x20000 && $c <= 0x2a6d6) || # cjk unified ideographs ext B
+       # ($c >= 0x2f800 && $c <= 0x2fa1d)) { #cjk compatibility ideographs supplement
         # CJK character
         push (@$out, 0x200b) unless $space;

gsdl/trunk/runtime-src/src/recpt/querytools.cpp

@@ -305 +305 @@
       formattedstring.push_back(' ');
     } else if (segment) {
-      if ((*here >= 0x2e80 && *here <= 0xfa6a) ||
-          (*here >= 0x20000 && *here <= 0x2a6d6) ||
-      (*here >= 0x2f800 && *here <= 0x2fa1d)) {
+      if ((*here >= 0x2e80 && *here <= 0xd7a3) ||
+      ( *here >= 0xf900 && *here <= 0xfa6a)) {
+    /* text_t not big enough to handle these. */
+    /*    (*here >= 0x20000 && *here <= 0x2a6d6) ||
+      (*here >= 0x2f800 && *here <= 0x2fa1d)) { */
     
     // CJK character