Changeset 1927

Timestamp:

2001-02-08T13:17:33+13:00 (23 years ago)

Author:

sjboddie

Message:

Fixed a bug in the C++ encoding support - 8 bit encodings like windows-1251
were being treated as 16 bit encodings in some places

Location:

trunk/gsdl

Files:

• etc/main.cfg (modified) (2 diffs)
• lib/gsdlunicode.cpp (modified) (4 diffs)
• lib/gsdlunicode.h (modified) (5 diffs)
• src/recpt/receptionist.cpp (modified) (2 diffs)

trunk/gsdl/etc/main.cfg

@@ -89 +89 @@
 #              option is mandatory for all encoding lines except the
 #              special case for utf8.
+# multibyte -- This optional argument should be set for all encodings that use
+#              multibyte characters.
 
 # The utf8 encoding is handled internally and doesn't require a map file.
@@ -125 +127 @@
 
 # CJK encodings (note that Shift-JIS Japanese isn't currently supported)
-Encoding shortname=gbk "longname=Chinese Simplified (GBK)" map=gbk.ump
-Encoding shortname=big5 "longname=Chinese Traditional (Big5)" map=big5.ump
-Encoding shortname=euc-jp "longname=Japanese (EUC)" map=euc_jp.ump
-Encoding shortname=euc-kr "longname=Korean (UHC)" map=uhc.ump
+Encoding shortname=gbk "longname=Chinese Simplified (GBK)" map=gbk.ump multibyte
+Encoding shortname=big5 "longname=Chinese Traditional (Big5)" map=big5.ump multibyte
+Encoding shortname=euc-jp "longname=Japanese (EUC)" map=euc_jp.ump multibyte
+Encoding shortname=euc-kr "longname=Korean (UHC)" map=uhc.ump multibyte
 
 

trunk/gsdl/lib/gsdlunicode.cpp

@@ -384 +384 @@
 
 mapinconvertclass::mapinconvertclass () {
+  multibyte = 0;
   mapbuflen = 0;
 }
@@ -425 +426 @@
       // got a complete character
       if (charlen == 1) {
-    // ascii character
-    output.push_back (mapbuf[0]);
+    if (mapbuf[0] < 0x80) {
+      // ascii character
+      output.push_back (mapbuf[0]);
+    } else {
+      output.push_back (converter.convert((unsigned short)mapbuf[0]));
+    }
 
       } else {
@@ -448 +453 @@
 
 mapoutconvertclass::mapoutconvertclass () {
+  multibyte = 0;
   mapbuflen=0; 
   mapbufhere=0;
@@ -500 +506 @@
     } else {
       outc = converter.convert (*texthere);
-      mapbuf[0] = (unsigned char)(outc >> 8);
-      mapbuf[1] = (unsigned char)(outc & 0xff);
-      mapbuflen = 2;
+      if (multibyte) {
+        mapbuf[0] = (unsigned char)(outc >> 8);
+        mapbuf[1] = (unsigned char)(outc & 0xff);
+        mapbuflen = 2;
+      } else {
+        mapbuf[0] = outc;
+        mapbuflen = 1;
+      }
     }
       }

trunk/gsdl/lib/gsdlunicode.h

@@ -153 +153 @@
   };
 
+  void set_multibyte (int new_multibyte) {multibyte = new_multibyte;};
+
   void reset ();
   void convert (text_t &output, status_t &status);
@@ -160 +162 @@
   unsigned char mapbuf[MAXMAPCHARLEN];
   size_t mapbuflen;
+  int multibyte;
 
   // note: multiple instances of mapinconvert class are expensive
@@ -172 +175 @@
     if (mapbuflen == 0) return 0;
     if (mapbuf[0] < 0x80) return 1;
+    if (!multibyte) return 1;
     return 2;
   }
@@ -194 +198 @@
   };
 
+  void set_multibyte (int new_multibyte) {multibyte = new_multibyte;};
+
   void reset ();
   void convert (char *output, size_t maxlen, 
@@ -202 +208 @@
   size_t mapbuflen;
   size_t mapbufhere;
+  int multibyte;
 
   mapconvert converter;

trunk/gsdl/src/recpt/receptionist.cpp

@@ -1300 +1300 @@
 
   text_t subkey, subvalue, shortname, longname, mapfile;
+  int multibyte = 0;
   text_t::const_iterator cfglinesub_here;
   text_tarray::const_iterator cfgline_here = cfgline.begin();
   text_tarray::const_iterator cfgline_end = cfgline.end();
   while (cfgline_here != cfgline_end) {
-    cfglinesub_here = getdelimitstr((*cfgline_here).begin(),
-                    (*cfgline_here).end(), '=', subkey);
-    if (subkey == "shortname") {
-      shortname = substr (cfglinesub_here, (*cfgline_here).end());
-    } else if (subkey == "longname") {
-      longname = substr (cfglinesub_here, (*cfgline_here).end());
-    } else if (subkey == "map") {
-      mapfile = substr (cfglinesub_here, (*cfgline_here).end());
+    if (*cfgline_here == "multibyte") {
+      multibyte = 1;
+    } else {
+      cfglinesub_here = getdelimitstr((*cfgline_here).begin(),
+                      (*cfgline_here).end(), '=', subkey);
+      if (subkey == "shortname") {
+    shortname = substr (cfglinesub_here, (*cfgline_here).end());
+      } else if (subkey == "longname") {
+    longname = substr (cfglinesub_here, (*cfgline_here).end());
+      } else if (subkey == "map") {
+    mapfile = substr (cfglinesub_here, (*cfgline_here).end());
+      }
     }
     cfgline_here++;
@@ -1345 +1350 @@
       mapinconvertclass *mapinconvert = new mapinconvertclass();
       mapinconvert->setmapfile (to_uc_map, 0x003F);
+      mapinconvert->set_multibyte (multibyte);
       mapoutconvertclass *mapoutconvert = new mapoutconvertclass();
       mapoutconvert->setmapfile (from_uc_map, 0x3F);
+      mapoutconvert->set_multibyte (multibyte);
       mapoutconvert->set_rzws(1);
       add_converter (shortname, mapinconvert, mapoutconvert);