Changeset 94 for trunk/gsdl

Timestamp:

1999-01-04T16:32:21+13:00 (25 years ago)

Author:

rjmcnab

Message:

Wrote general map file based in and out converters. Fixed bugs related
to Chinese charater searching. text_t now has a encoding attribute. Added
an encoding option to the preferences.

Location:

trunk/gsdl

Files:

• lib/Makefile (modified) (4 diffs)
• lib/gsdlunicode.cpp (modified) (5 diffs)
• lib/gsdlunicode.h (modified) (3 diffs)
• lib/text_t.cpp (modified) (11 diffs)
• lib/text_t.h (modified) (5 diffs)
• src/library/Makefile (modified) (4 diffs)
• src/library/cgiargs.cpp (modified) (3 diffs)
• src/library/libinterface.cpp (modified) (18 diffs)
• src/library/libinterface.h (modified) (2 diffs)
• src/library/mgq.c (modified) (2 diffs)
• src/library/mgsearch.cpp (modified) (4 diffs)

trunk/gsdl/lib/Makefile

@@ -26 +26 @@
 CC = gcc
 CCFLAGS =
-DEFS = -O2 -g
+DEFS = -g
 RANLIB = ranlib
 INCLUDES = -I../packages/mg-1.3d/lib
@@ -49 +49 @@
 OBJECTS = text_t.o display.o cfgread.o fileutil.o gsdlunicode.o
  
-all : gsdllib.a
+all: gsdllib.a
 
-gsdllib.a : $(OBJECTS)
+gsdllib.a: $(OBJECTS)
     rm -f gsdllib.a
     $(AR) cru gsdllib.a $(OBJECTS)
@@ -59 +59 @@
     rm -f $(OBJECTS) gsdllib.a
 
+install:
+
 depend:
     makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
@@ -65 +67 @@
 
 text_t.o: text_t.h
-display.o: display.h text_t.h
+display.o: display.h text_t.h gsdlunicode.h
 cfgread.o: cfgread.h text_t.h
 fileutil.o: fileutil.h text_t.h
+gsdlunicode.o: gsdlunicode.h text_t.h ../packages/mg-1.3d/lib/unitool.h
+gsdlunicode.o: fileutil.h

trunk/gsdl/lib/gsdlunicode.cpp

@@ -9 +9 @@
 #include <stdio.h>
 
-
-mapdata_t::mapdata_t () {
-  int i;
-
-  // reset all the map ptrs to be NULL
-  for (i=0; i<256; i++) {
-    ptrs[i] = (unsigned short *)NULL;
-  }
-
-  // say nothing has been loaded
-  loaded = false;
-}
-
-
-mapconvert::mapconvert (const text_t &thegsdlhome, const text_t &theencoding, 
-            unsigned short theabsentc) {
-  gsdlhome = thegsdlhome;
-  encoding = theencoding;
-  absentc = theabsentc;
-}
-
-unsigned short mapconvert::convert (unsigned short c) {
-  if (!loadmapfile()) return absentc;
-
-  if (c == 0) return 0; // 0 always maps to 0...
-
-  unsigned short n1 = c >> 8;
-  unsigned short n2 = c & 0xff;
-
-  unsigned short *arrptr = mapdata.ptrs[n1];
-  if (arrptr == (unsigned short *)NULL) return absentc;
-
-  if (arrptr[n2] == 0) return absentc;
-  return arrptr[n2];
-}
-
-text_t mapconvert::convert (const text_t &instr) {
-  if (!loadmapfile()) return absentc;
-
-  text_t outstr;
-  text_t::const_iterator here = instr.begin();
-  text_t::const_iterator end = instr.end();
-
-  while (here != end) {
-    outstr.push_back(this->convert(*here));
-    here++;
-  }
-  
-  return outstr;
-}
-
-bool mapconvert::loadmapfile () {
-  FILE *mapfilein = (FILE *)NULL;
-
-  // check to see if the mapfile has been already loaded
-  if (mapdata.loaded) return true;
-
-  // open the map file
-  text_t filename = filename_cat (gsdlhome, "unicode");
-  filename = filename_cat (filename, encoding);
-  filename += ".ump";
-  char *cfilename = filename.getcstr();
-  if (cfilename == (char *)NULL) return false;
-  mapfilein = fopen(cfilename, "rb");
-  delete cfilename;
-
-  if (mapfilein == (FILE *)NULL) return false;
-
-  unsigned char c, n1, n2;
-  unsigned short *arrptr;
-  int i;
-  c = fgetc (mapfilein);
-  while (!feof (mapfilein)) {
-    if (mapdata.ptrs[c] == (unsigned short *)NULL) {
-      // allocate a new array
-      arrptr = new unsigned short[256];
-      mapdata.ptrs[c] = arrptr;
-    } else arrptr = mapdata.ptrs[c];
-
-    // clear the array
-    for (i=0; i<256; i++) arrptr[i] = 0;
-    
-    // read in this block
-    n1 = fgetc (mapfilein);
-    n2 = fgetc (mapfilein);
-    i=0;
-    while (!feof (mapfilein)) {
-      arrptr[i] = ((unsigned short)n1 << 8) | (unsigned short)n2;
-
-      i++;
-      if (i >= 256) break;
-      n1 = fgetc (mapfilein);
-      n2 = fgetc (mapfilein);
-    }
-
-    c = fgetc (mapfilein);
-  }
-
-  mapdata.loaded = true;
-
-  return true;
-}
-
-void mapconvert::unloadmapfile () {
-  if (!mapdata.loaded) return;
-
-  int i;
-  for (i=0; i<256; i++) {
-    if (mapdata.ptrs[i] != (unsigned short *)NULL) {
-      delete [] mapdata.ptrs[i];
-      mapdata.ptrs[i] = (unsigned short *)NULL;
-    }
-  }
-
-  mapdata.loaded = false;
-}
 
 
@@ -189 +73 @@
 
 void utf8inconvertclass::convert (text_t &output, status_t &status) {
+  output.clear();
+
   if (start == NULL || len == 0) {
     if (utf8buflen == 0) status = finished;
@@ -261 +147 @@
 
 
-
-gbinconvertclass::gbinconvertclass (const text_t &gsdlhome) {
-  gbbuflen = 0;
-  gb2unimap = new mapconvert (gsdlhome, "gbku", 0x25a1);
-}
-
-void gbinconvertclass::reset () {
-  start = NULL;
-  len = 0;
-  gbbuflen=0;
-}
-
-void gbinconvertclass::convert (text_t &output, status_t &status) {
-  if (start == NULL || len == 0 || gb2unimap == NULL) {
-    if (gbbuflen == 0) status = finished;
-    else status = stopped;
-    return;
-  }
-
-  // don't want any funny sign conversions happening
-  unsigned char *here = (unsigned char *)start;
-
-  size_t charlen = getgbcharlen ();
-  unsigned short c;
-  size_t realcharlen;
-  while (len > 0) {
-    if (charlen == 0) {
-      // start parsing a new character
-      gbbuflen = 0;
-      gbbuf[gbbuflen++] = *here;
-      ++here;
-      --len;
-      charlen = getgbcharlen ();
-
-    } else if (gbbuflen < charlen) {
-      // assumes charlen is always less than MAXGBCHARLEN
-      gbbuf[gbbuflen++] = *here;
-      ++here;
-      --len;
-    }
-
-    if (gbbuflen == charlen) {
-      // got a complete character
-      if (charlen == 1) {
-    // ascii character
-    output.push_back (gbbuf[0]);
-
-      } else {
-    // two byte character
-    output.push_back (gb2unimap->convert(((unsigned short)gbbuf[0] << 8) |
-                         (unsigned short)gbbuf[1]));
-      }
-
-      gbbuflen = 0;
-      charlen = 0;
-    }
-  }
-
-  start = (char *)here; // save current position
-
-  if (gbbuflen == 0) status = finished;
-  else status = stopped;
-}
-
-// returns the length that the current contents of the 
-// gbbuf should be
-size_t gbinconvertclass::getgbcharlen () {
-  if (gbbuflen == 0) return 0;
-
-  // one byte character
-  if (gbbuf[0] < 0x80) return 1;
-
-  // other characters are two byte
-  return 2;
-}
-
-
-
-
-
-
 void utf8outconvertclass::reset () {
   input = NULL;
@@ -397 +202 @@
 
 
-gboutconvertclass::gboutconvertclass (const text_t &gsdlhome){
-  gbbuflen=0; 
-  gbbufhere=0;
-  uni2gbmap = new mapconvert (gsdlhome, "ugbk", 0xa1f5);
-}
-
-void gboutconvertclass::reset () {
+
+
+mapdata_t::mapdata_t () {
+  int i;
+
+  // reset all the map ptrs to be NULL
+  for (i=0; i<256; i++) {
+    ptrs[i] = (unsigned short *)NULL;
+  }
+
+  // say nothing has been loaded
+  loaded = false;
+}
+
+
+mapconvert::mapconvert () {
+  absentc = 0;
+}
+
+
+// loadmapfile should be called before any conversion is done
+bool mapconvert::loadmapfile (const text_t &thegsdlhome, 
+                  const text_t &theencoding, 
+                  unsigned short theabsentc) {
+  FILE *mapfilein = (FILE *)NULL;
+
+  // check to see if the mapfile has been already loaded
+  if (mapdata.loaded && gsdlhome == thegsdlhome &&
+      encoding == theencoding && absentc == theabsentc) 
+    return true;
+
+  unloadmapfile ();
+  gsdlhome = thegsdlhome;
+  encoding = theencoding;
+  absentc = theabsentc;
+
+  // open the map file
+  text_t filename = filename_cat (gsdlhome, "unicode");
+  filename = filename_cat (filename, encoding);
+  filename += ".ump";
+  char *cfilename = filename.getcstr();
+  if (cfilename == (char *)NULL) return false;
+  mapfilein = fopen(cfilename, "rb");
+  delete cfilename;
+
+  if (mapfilein == (FILE *)NULL) return false;
+
+  unsigned char c, n1, n2;
+  unsigned short *arrptr;
+  int i;
+  c = fgetc (mapfilein);
+  while (!feof (mapfilein)) {
+    if (mapdata.ptrs[c] == (unsigned short *)NULL) {
+      // allocate a new array
+      arrptr = new unsigned short[256];
+      mapdata.ptrs[c] = arrptr;
+    } else arrptr = mapdata.ptrs[c];
+
+    // clear the array
+    for (i=0; i<256; i++) arrptr[i] = 0;
+    
+    // read in this block
+    n1 = fgetc (mapfilein);
+    n2 = fgetc (mapfilein);
+    i=0;
+    while (!feof (mapfilein)) {
+      arrptr[i] = ((unsigned short)n1 << 8) | (unsigned short)n2;
+
+      i++;
+      if (i >= 256) break;
+      n1 = fgetc (mapfilein);
+      n2 = fgetc (mapfilein);
+    }
+
+    c = fgetc (mapfilein);
+  }
+
+  mapdata.loaded = true;
+
+  return true;
+}
+
+void mapconvert::unloadmapfile () {
+  if (!mapdata.loaded) return;
+
+  int i;
+  for (i=0; i<256; i++) {
+    if (mapdata.ptrs[i] != (unsigned short *)NULL) {
+      delete [] mapdata.ptrs[i];
+      mapdata.ptrs[i] = (unsigned short *)NULL;
+    }
+  }
+
+  mapdata.loaded = false;
+}
+
+
+unsigned short mapconvert::convert (unsigned short c) {
+  if (!mapdata.loaded) return absentc;
+
+  if (c == 0) return 0; // 0 always maps to 0...
+
+  unsigned short n1 = c >> 8;
+  unsigned short n2 = c & 0xff;
+
+  unsigned short *arrptr = mapdata.ptrs[n1];
+  if (arrptr == (unsigned short *)NULL) return absentc;
+
+  if (arrptr[n2] == 0) return absentc;
+  return arrptr[n2];
+}
+
+text_t mapconvert::convert (const text_t &instr) {
+  if (!mapdata.loaded) return absentc;
+
+  text_t outstr;
+  text_t::const_iterator here = instr.begin();
+  text_t::const_iterator end = instr.end();
+
+  while (here != end) {
+    outstr.push_back(this->convert(*here));
+    here++;
+  }
+  
+  return outstr;
+}
+
+
+
+
+mapinconvertclass::mapinconvertclass () {
+  mapbuflen = 0;
+}
+
+void mapinconvertclass::reset () {
+  start = NULL;
+  len = 0;
+  mapbuflen=0;
+}
+
+void mapinconvertclass::convert (text_t &output, status_t &status) {
+  output.clear();
+
+  if (start == NULL || len == 0) {
+    if (mapbuflen == 0) status = finished;
+    else status = stopped;
+    return;
+  }
+
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+
+  size_t charlen = getmapcharlen ();
+  unsigned short c;
+  size_t realcharlen;
+  while (len > 0) {
+    if (charlen == 0) {
+      // start parsing a new character
+      mapbuflen = 0;
+      mapbuf[mapbuflen++] = *here;
+      ++here;
+      --len;
+      charlen = getmapcharlen ();
+
+    } else if (mapbuflen < charlen) {
+      // assumes charlen is always less than MAXMAPCHARLEN
+      mapbuf[mapbuflen++] = *here;
+      ++here;
+      --len;
+    }
+
+    if (mapbuflen == charlen) {
+      // got a complete character
+      if (charlen == 1) {
+    // ascii character
+    output.push_back (mapbuf[0]);
+
+      } else {
+    // two byte character
+    output.push_back (converter.convert(((unsigned short)mapbuf[0] << 8) |
+                        (unsigned short)mapbuf[1]));
+      }
+
+      mapbuflen = 0;
+      charlen = 0;
+    }
+  }
+
+  start = (char *)here; // save current position
+
+  if (mapbuflen == 0) status = finished;
+  else status = stopped;
+}
+
+
+
+mapoutconvertclass::mapoutconvertclass () {
+  mapbuflen=0; 
+  mapbufhere=0;
+}
+
+void mapoutconvertclass::reset () {
   input = NULL;
   outs = NULL;
-  gbbuflen = 0;
-  gbbufhere = 0;
+  mapbuflen = 0;
+  mapbufhere = 0;
 }
 
 // note that convert does not null-terminate the
 // output array of characters
-void gboutconvertclass::convert (char *output, size_t maxlen, 
+void mapoutconvertclass::convert (char *output, size_t maxlen, 
                  size_t &len, status_t &status) {
   unsigned short outc;
 
-  if (input == NULL || output == NULL || uni2gbmap == NULL) {
-    if (gbbuflen == 0) status = finished;
+  if (input == NULL || output == NULL) {
+    if (mapbuflen == 0) status = finished;
     else status = unfinished;
     return;
@@ -428 +428 @@
   while (len < maxlen) {
     // empty the contents of the internal buffer
-    if (gbbuflen > 0) {
-      while (len < maxlen && gbbufhere < gbbuflen) {
-    *uoutput = gbbuf[gbbufhere];
+    if (mapbuflen > 0) {
+      while (len < maxlen && mapbufhere < mapbuflen) {
+    *uoutput = mapbuf[mapbufhere];
     uoutput++;
     len++;
-    gbbufhere++;
-      }
-
-      if (gbbufhere == gbbuflen) {
-    gbbufhere = 0;
-    gbbuflen = 0;
+    mapbufhere++;
+      }
+
+      if (mapbufhere == mapbuflen) {
+    mapbufhere = 0;
+    mapbuflen = 0;
       }
     }
 
     // fill up the buffer with the next character
-    if (gbbuflen == 0) {
+    if (mapbuflen == 0) {
       if (texthere == textend) break; // finished!
       if (!rzws || (*texthere != 0x200b)) {
     if (*texthere < 0x80) {
-      gbbuf[0] = (unsigned char)*texthere;
-      gbbuflen = 1;
+      mapbuf[0] = (unsigned char)*texthere;
+      mapbuflen = 1;
     } else {
-      outc = uni2gbmap->convert (*texthere);
-      gbbuf[0] = (unsigned char)(outc >> 8);
-      gbbuf[1] = (unsigned char)(outc & 0xff);
-      gbbuflen = 2;
+      outc = converter.convert (*texthere);
+      mapbuf[0] = (unsigned char)(outc >> 8);
+      mapbuf[1] = (unsigned char)(outc & 0xff);
+      mapbuflen = 2;
     }
       }
 
       texthere++;
-      gbbufhere = 0;
+      mapbufhere = 0;
     }
   }
   
-  if (texthere == textend && gbbuflen == 0) status = finished;
+  if (texthere == textend && mapbuflen == 0) status = finished;
   else status = unfinished;
 }

trunk/gsdl/lib/gsdlunicode.h

@@ -3 +3 @@
 
 #include "text_t.h"
-
-// mapdata_t is used by mapconvert to hold the map file data
-class mapdata_t {
-public:
-  mapdata_t();
-  bool loaded;
-  unsigned short *ptrs[256];
-};
-
-
-// mapconvert is used in situations where conversion is best
-// done using a map file. The mapfile should reside in 
-// gsdlhome/unicode.
-class mapconvert {
-public:
-  mapconvert (const text_t &thegsdlhome, const text_t &theencoding, 
-          unsigned short theabsentc);
-  ~mapconvert () {unloadmapfile();};
-
-  unsigned short convert (unsigned short c);
-  text_t convert (const text_t &instr);
-
-protected:
-  text_t gsdlhome;
-  text_t encoding;
-  unsigned short absentc;
-  mapdata_t mapdata;
-
-  bool loadmapfile ();
-  void unloadmapfile ();
-};
-
-
 
 
@@ -65 +32 @@
   // utf8buf should be
   size_t getutf8charlen ();
-};
-
-
-#define MAXGBCHARLEN 2
-
-// convert from a gb char stream to the unicode text_t class
-class gbinconvertclass : public inconvertclass {
-public:
-  gbinconvertclass(const text_t &gsdlhome);
-  ~gbinconvertclass () {delete gb2unimap;};
-  void reset ();
-  void convert (text_t &output, status_t &status);
-
-protected:
-  // buffer to hold unconverted characters in a stream
-  unsigned char gbbuf[MAXGBCHARLEN];
-  size_t gbbuflen;
-
-  // note: multiple instances of gbinconvert class are expensive
-  // as each will have its own copy of the map file data. This
-  // could be reduced by making gb2unimap static, but then it
-  // wouldn't be thread safe.
-  mapconvert *gb2unimap;
-
-  // returns the length that the current contents of the 
-  // gbbuf should be
-  size_t getgbcharlen ();
 };
 
@@ -129 +69 @@
 
 
+// mapdata_t is used by mapconvert to hold the map file data
+class mapdata_t {
+public:
+  mapdata_t();
+  bool loaded;
+  unsigned short *ptrs[256];
+};
+
+// mapconvert is used in situations where conversion is best
+// done using a map file. The mapfile should reside in 
+// gsdlhome/unicode.
+class mapconvert {
+public:
+  mapconvert ();
+  ~mapconvert () {unloadmapfile();};
+
+  // loadmapfile should be called before any conversion is done
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+            unsigned short theabsentc);
+  void unloadmapfile ();
+
+  unsigned short convert (unsigned short c);
+
+  // note that this version of convert has different semantics to
+  // the convertclass version.
+  text_t convert (const text_t &instr);
+
+protected:
+  text_t gsdlhome;
+  text_t encoding;
+  unsigned short absentc;
+  mapdata_t mapdata;
+};
 
 
-// Convert from a text_t class to a gb char stream
-class gboutconvertclass : public rzwsoutconvertclass {
+
+#define MAXMAPCHARLEN 2
+
+// convert from a gb char stream to the unicode text_t class
+class mapinconvertclass : public inconvertclass {
 public:
-  gboutconvertclass (const text_t &gsdhome);
-  ~gboutconvertclass () {delete uni2gbmap;};
+  mapinconvertclass();
+
+  // loadmapfile should be called before any conversion takes
+  // place
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+            unsigned short theabsentc) {
+    return converter.loadmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
   void reset ();
-  // note that convert does not null-terminate the
-  // output array of characters
+  void convert (text_t &output, status_t &status);
+
+protected:
+  // buffer to hold unconverted characters in a stream
+  unsigned char mapbuf[MAXMAPCHARLEN];
+  size_t mapbuflen;
+
+  // note: multiple instances of mapinconvert class are expensive
+  // as each will have its own copy of the map file data. This
+  // could be reduced by making map2unimap static, but then it
+  // wouldn't be thread safe.
+  mapconvert converter;
+
+  // returns the length that the current contents of the 
+  // mapbuf should be
+  inline size_t getmapcharlen () {
+    if (mapbuflen == 0) return 0;
+    if (mapbuf[0] < 0x80) return 1;
+    return 2;
+  }
+};
+
+
+// Convert from a text_t class to a map char stream
+class mapoutconvertclass : public rzwsoutconvertclass {
+public:
+  mapoutconvertclass ();
+
+  // loadmapfile should be called before any conversion takes
+  // place
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+            unsigned short theabsentc) {
+    return converter.loadmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
+  void reset ();
   void convert (char *output, size_t maxlen, 
         size_t &len, status_t &status);
 
 protected:
-  unsigned char gbbuf[MAXGBCHARLEN];
-  size_t gbbuflen;
-  size_t gbbufhere;
+  unsigned char mapbuf[MAXMAPCHARLEN];
+  size_t mapbuflen;
+  size_t mapbufhere;
 
-  mapconvert *uni2gbmap;
+  mapconvert converter;
 };
 

trunk/gsdl/lib/text_t.cpp

@@ -24 +24 @@
 /*
    $Log$
+   Revision 1.2  1999/01/04 03:32:17  rjmcnab
+
+   Wrote general map file based in and out converters. Fixed bugs related
+   to Chinese charater searching. text_t now has a encoding attribute. Added
+   an encoding option to the preferences.
+
    Revision 1.1  1998/11/17 09:11:29  rjmcnab
 
@@ -70 +76 @@
 
 #include "text_t.h"
+#include "unitool.h"
 
 ////////////////////////////////////
@@ -77 +84 @@
 text_t::text_t () 
 {
+  setencoding(0);
   clear ();
 }
@@ -82 +90 @@
 text_t::text_t (int i) 
 {
+  setencoding(0);
   clear ();
   appendint (i);
@@ -87 +96 @@
 
 text_t::text_t (char *s) 
-{
+{ 
+  setencoding(0);
   clear ();
   appendcstr (s);
@@ -223 +233 @@
   while (ithere != itend)
     {
-      if (*ithere >= 256) cstr[len] = ' ';
-      else cstr[len] = (*ithere);
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+    // put a space or a question mark depending on what
+    // the character is. Question marks tell the user that
+    // they are missing some information.
+    if (is_unicode_space (*ithere)) cstr[len] = ' ';
+    else cstr[len] = '?';
+      }
       len++;
       ithere++;
@@ -241 +257 @@
   while (ithere != itend)
     {
-      if (*ithere >= 256) cstr[len] = ' ';
-      else cstr[len] = (*ithere);
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+    // put a space or a question mark depending on what
+    // the character is. Question marks tell the user that
+    // they are missing some information.
+    if (is_unicode_space (*ithere)) cstr[len] = ' ';
+    else cstr[len] = '?';
+      }
       len++;
       ithere++;
@@ -425 +447 @@
 void inconvertclass::convert (text_t &output, status_t &status)
 {
+  output.clear();
+
   if (start == NULL || len == 0)
     {
@@ -442 +466 @@
   start = (char *)here; // save current position
   status = finished;
+}
+
+// will treat the text_t as a 8-bit string and convert
+// it to a 16-bit string using the about convert method.
+text_t inconvertclass::convert (const text_t &t) {
+  text_t out;
+  text_t tmpout;
+  status_t status;
+  text_t::const_iterator here = t.begin();
+  text_t::const_iterator end = t.end();
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+ 
+  while (here != end) {
+    while (here != end && cbuflen < 256) {
+      cbuf[cbuflen++] = (unsigned char)(*here & 0xff);
+      here++;
+    }
+
+    if (cbuflen > 0) {
+      setinput ((char *)cbuf, cbuflen);
+      status = unfinished;
+      while (status == unfinished) {
+    convert (tmpout, status);
+    out += tmpout;
+      }
+      cbuflen = 0;
+    }
+  }
+
+  out.setencoding (0); // unicode
+
+  return out;
 }
 
@@ -498 +555 @@
     {
       if (*texthere < 256) *uoutput = (unsigned char)(*texthere);
-      else *uoutput = 32; // put a space where a char >= 256 exists
+      else {
+    // put a space or a question mark depending on what
+    // the character is. Question marks tell the user that
+    // they are missing some information.
+    if (is_unicode_space (*texthere)) *uoutput = ' ';
+    else *uoutput = '?';
+      }
       ++uoutput;
       ++len;
@@ -508 +571 @@
 }
 
+// will convert the 16-bit string to a 8-bit stream
+// and place the result in a text_t. This method uses
+// the above convert function.
+text_t outconvertclass::convert (const text_t &t) {
+  text_t out;
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+  status_t status = unfinished;
+
+  setinput ((text_t *)&t); // discard constant
+  while (status == unfinished) {
+    convert ((char *)cbuf, 256, cbuflen, status);
+    out.appendcarr ((char *)cbuf, cbuflen);
+  }
+
+  out.setencoding (1); // other encoding
+  
+  return out;
+}
+
+
 void outconvertclass::setostream (ostream *theouts)
 {

trunk/gsdl/lib/text_t.h

@@ -75 +75 @@
 protected:
   usvector text;
+  unsigned short encoding; // 0 = unicode, 1 = other
   
 public:
@@ -82 +83 @@
   text_t (char *s); // assumed to be a normal c string
 
+  void setencoding (unsigned short theencoding) {encoding=theencoding;};
+  unsigned short getencoding () {return encoding;};
+
   // basic container support
   iterator begin () {return text.begin();}
@@ -92 +96 @@
   void push_back(unsigned short c) {text.push_back(c);}
   void pop_back() {text.pop_back();}
-  text_t &operator=(const text_t &x) {text=x.text;return *this;}
+  text_t &operator=(const text_t &x) {text=x.text; encoding=x.encoding; return *this;}
   reference operator[](size_type n) {return text[n];};
   const_reference operator[](size_type n) const {return text[n];};
@@ -255 +259 @@
   void reset ();
   void setinput (char *thestart, size_t thelen);
+
+  // output will be cleared before the conversion
   virtual void convert (text_t &output, status_t &status);
+
+  // will treat the text_t as a 8-bit string and convert
+  // it to a 16-bit string using the about convert method.
+  text_t convert (const text_t &t);
 
 protected:
@@ -290 +300 @@
             size_t &len, status_t &status);
 
+  // will convert the 16-bit string to a 8-bit stream
+  // and place the result in a text_t. This method uses
+  // the above convert function.
+  text_t convert (const text_t &t);
+
   void setostream (ostream *theouts);
   ostream *getostream ();

trunk/gsdl/src/library/Makefile

@@ -25 +25 @@
 AR = ar
 CC = gcc
-CCFLAGS = -O2 -g
+CCFLAGS = -g
 DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DUSE_FASTCGI
 RANLIB = ranlib
@@ -73 +73 @@
     rm -f $(OBJECTS)
 
+install:
+
 depend:
     makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
@@ -80 +82 @@
 # DO NOT DELETE
 
-browse.o: browse.h ../../lib/text_t.h gdbmclass.h locateinfo.h cgiargs.h
-gdbmclass.o: ../../lib/text_t.h gdbmclass.h locateinfo.h
+browse.o: browse.h ../../lib/text_t.h gdbmclass.h cgiargs.h
+gdbmclass.o: ../../lib/text_t.h gdbmclass.h
+gdbmclass.o: ../../packages/mg-1.3d/lib/unitool.h ../../lib/gsdlunicode.h
+gdbmclass.o: ../../lib/fileutil.h
 cgiargs.o: cgiargs.h ../../lib/text_t.h
 querycache.o: querycache.h ../../lib/text_t.h queryinfo.h
@@ -106 +110 @@
 queryinfo.o: queryinfo.h ../../lib/text_t.h
 libinterface.o: libinterface.h browse.h ../../lib/text_t.h gdbmclass.h
-libinterface.o: locateinfo.h cgiargs.h queryinfo.h mgsearch.h querycache.h
-libinterface.o: ../../lib/display.h
+libinterface.o: cgiargs.h queryinfo.h mgsearch.h querycache.h
+libinterface.o: ../../lib/display.h ../../lib/gsdlunicode.h
+libinterface.o: ../../lib/fileutil.h ../../lib/cfgread.h
 mgsearch.o: mgq.h mgsearch.h ../../lib/text_t.h querycache.h queryinfo.h
-mgsearch.o: locateinfo.h
-locateinfo.o: locateinfo.h ../../lib/text_t.h
-cgiwrap.o: libinterface.h browse.h ../../lib/text_t.h gdbmclass.h
-cgiwrap.o: locateinfo.h cgiargs.h queryinfo.h mgsearch.h querycache.h
-cgiwrap.o: ../../lib/display.h
+mgsearch.o: locateinfo.h ../../lib/gsdlunicode.h
+mgsearch.o: ../../packages/mg-1.3d/lib/unitool.h
+locateinfo.o: locateinfo.h ../../lib/text_t.h ../../lib/fileutil.h
+cgiwrap.o: libinterface.h browse.h ../../lib/text_t.h gdbmclass.h cgiargs.h
+cgiwrap.o: queryinfo.h mgsearch.h querycache.h ../../lib/display.h
+cgiwrap.o: ../../lib/gsdlunicode.h

trunk/gsdl/src/library/cgiargs.cpp

@@ -1 +1 @@
 #include "cgiargs.h"
-
+#include "gsdlunicode.h"
 
   
@@ -64 +64 @@
 ostream &operator<<(ostream &outs, const cgiargsclass &args)
 {
-  outconvertclass text_t2ascii;
+  utf8outconvertclass text_t2utf8;
   cgiargsclass::const_iterator here = args.begin ();
   cgiargsclass::const_iterator end = args.end ();
@@ -72 +72 @@
   while (here != end)
     {
-      outs << text_t2ascii << " \"" << (*here).first << "\"=\"" << 
+      outs << text_t2utf8 << " \"" << (*here).first << "\"=\"" << 
     (*here).second << "\"\n";
       here++;

trunk/gsdl/src/library/libinterface.cpp

@@ -9 +9 @@
 #include "cfgread.h"
 #include "gsdlunicode.h"
+#include "unitool.h"
 
 #include <assert.h>
@@ -107 +108 @@
       // convert %xx and + to their appropriate equivalents
       decode (value);
+      value.setencoding(1); // other encoding
       // store this key=value pair
       if (!key.empty()) args.setarg (key, value);
@@ -112 +114 @@
 }
 
-text_t cgisafe (text_t &intext)
+text_t cgisafe (const text_t &intext)
 {
   text_t outtext;
 
-  text_t::iterator here = intext.begin ();
-  text_t::iterator end = intext.end ();
+  text_t::const_iterator here = intext.begin ();
+  text_t::const_iterator end = intext.end ();
   unsigned short c;
   text_t ttmp;
@@ -150 +152 @@
 libinterface::libinterface() {
   browse = NULL;
-  gbinconvert = NULL;
-  gboutconvert = NULL;
 }
 
@@ -277 +277 @@
   srand(time(NULL));
 
-  gbinconvert = new gbinconvertclass (gsdlhome);
-
   utf8outconvert.set_rzws(1);
-  gboutconvert = new gboutconvertclass (gsdlhome);
-  if (gboutconvert != NULL) gboutconvert->set_rzws(1);
+  gboutconvert.set_rzws(1);
 
   return collection_init(collection);
@@ -305 +302 @@
   check_args (args);
 
+  // get the input encoding
+  text_t &arg_w = args["w"];
+  inconvertclass *inconvert = NULL;
+  if (arg_w == "8") {
+    inconvert = &utf8inconvert;
+  } else if (arg_w == "g") {
+    // The map files will only be loaded the first time they are
+    // needed. The loading is done here to reduce the memory load
+    // for collections which don't need to convert to GB.
+    gbinconvert.loadmapfile (gsdlhome, "gbku", 0x25a1);
+    inconvert = &gbinconvert;
+  } else {
+    inconvert = &asciiinconvert; // default
+  }
+
+  // see if the next page will have a different encoding
+  if (args.getarg("nw") != NULL) args["w"] = args["nw"];
+
+  // convert arguments which aren't in unicode to unicode
+  args_tounicode (args, *inconvert);
+
+  // remember the state of the compressed arguments
   lastcomparg = get_compressed_args (args);
-
   logout << args;
 
   // get the output encoding
-  text_t &arg_n = args["n"];
+  text_t &arg_nw = args["w"];
   outconvertclass *outconvert = NULL;
-  if (arg_n == "8") {
+  if (arg_nw == "8") {
     outconvert = &utf8outconvert;
-  } else if (arg_n == "g" && gboutconvert != NULL) {
-    outconvert = gboutconvert;
-  }
-  else outconvert = &asciioutconvert; // default
-
-  if (outconvert == NULL) return err;
+  } else if (arg_nw == "g") {
+    gboutconvert.loadmapfile (gsdlhome, "ugbk", 0xa1f5);
+    outconvert = &gboutconvert;
+  } else {
+    outconvert = &asciioutconvert; // default
+  }
+
 
   // dispatch the request
@@ -346 +365 @@
   argconfigstr = 
     "+a[p]"    // action: q=query, b=browse, t=targetdoc, p=page, a1=auxiliary
-    "n[]"     // encoding: w=western, 8=utf8, 7=utf7, g=GB2312, k=GBK
+    "w[]"      // encoding: w=western, 8=utf8, 7=utf7, g=GB2312, k=GBK
     "t[1]"     // query type: 0=boolean, 1=ranked
     "i[c]"     // index: c=chapter, p=paragraph, t=title, b=book
@@ -479 +498 @@
 {
   args.setarg("c", get_collection_name());
-  if (args["n"].empty()) args.setarg("n", cfg_info.defaultencoding);
-}
-
-
-void libinterface::define_general_macros (cgiargsclass &args, ostream &logout)
-{
+  if (args["w"].empty()) args.setarg("w", cfg_info.defaultencoding);
+}
+
+void libinterface::args_tounicode (cgiargsclass &args, 
+                   inconvertclass &inconvert) {
+  utf8outconvertclass text_t2utf8;
+  cgiargsclass::iterator here = args.begin();
+  cgiargsclass::iterator end = args.end();
+
+  while (here != end) {
+    if (here->second.getencoding() > 0) {
+      here->second = inconvert.convert(here->second);
+    }
+    
+    here++;
+  }
+}
+
+
+void libinterface::define_general_macros (cgiargsclass &args, outconvertclass &outconvert, 
+                      ostream &logout) {
   disp.setmacro("httpprefix", "Global", httpprefix);
   disp.setmacro("gwcgi", "Global", gwcgi);
 
-  disp.setmacro("collection", "Global", cgisafe(args["c"]));
+  disp.setmacro("numdocs", "Global", (int)cfg_info.numdocs);
+
+  disp.setmacro("collection", "Global", cgisafe(outconvert.convert(args["c"])));
   disp.setmacro("compressedoptions", "Global", get_compressed_args(args));
-  disp.setmacro("urlsafequerystring", "Global", cgisafe(args["q"]));
+  disp.setmacro("urlsafequerystring", "Global", cgisafe(outconvert.convert(args["q"])));
 
   // need to escape any quotes in querystring to prevent them upsetting the html
@@ -498 +534 @@
   while (here != end) {
     if (*here == '"') querystring += "&quot;";
+    else if (*here == '&') querystring += "&amp;";
+    else if (*here == '<') querystring += "&lt;";
+    else if (*here == '>') querystring += "&gt;";
     else querystring.push_back(*here);
     here ++;
@@ -503 +542 @@
   disp.setmacro("querystring", "Global", querystring);
 
-
   if (args.getintarg("x") == 0) disp.setmacro("notdetached", "Global", "1");
   if (args["d"][0] == 'T') disp.setmacro("istitle", "Global", "1");
@@ -517 +555 @@
 // prepare_page prepares to write out a page using the current 
 // page parameters and defines any general macros
-void libinterface::prepare_page (cgiargsclass &args, ostream &logout)
-{
+void libinterface::prepare_page (cgiargsclass &args, outconvertclass &outconvert,
+                 ostream &logout) {
   // get page parameters
   text_t pageparams = text_t("collection=") + args["c"];
@@ -527 +565 @@
 
   disp.openpage(pageparams, MACROPRECEDENCE);
-  define_general_macros(args, logout);
+  define_general_macros(args, outconvert, logout);
   define_collection_macros(args, logout);
 }
@@ -717 +755 @@
 
   // prepare to print out the page
-  prepare_page(args, logout);
+  prepare_page(args, outconvert, logout);
   define_query_macros(args, queryparams, queryresults, logout);
 
@@ -764 +802 @@
   gdbm_info info;
   
-  prepare_page(args, logout);
+  prepare_page(args, outconvert, logout);
   
   // get browse bar unless page has been detached 
@@ -806 +844 @@
   do_query(args, queryparams, queryresults, logout);
 
-  prepare_page(args, logout);
+  prepare_page(args, outconvert, logout);
   
   if (args["g"][1] == '0') {
@@ -976 +1014 @@
   text_t &arg_p = args["p"];
 
-  prepare_page(args, logout);
+  prepare_page(args, outconvert, logout);
 
   if (arg_p == "preferences")
@@ -999 +1037 @@
   text_t word, buffer;
   while (here != end) {
-    if (((*here >= 65) && (*here <= 90)) ||
-    ((*here >= 97) && (*here <= 122)) ||
-    ((*here >= '0') && (*here <= '9')) ||
-    ((*here >= 192) && (*here <= 214)) ||
-    ((*here >= 216) && (*here <= 246)) ||
-    ((*here >= 248) && (*here <= 255))) {
+    if (is_unicode_letdig(*here)) {
       // not word boundary
       word.push_back(*here);
@@ -1069 +1102 @@
   disp.setmacro ("stemoption", "preferences", stemoption);
 
+
+  // the encodingoption
+  text_t encodingoption;
+  const text_t &arg_w = args["w"];
+
+  encodingoption += "\n<select name=\"nw\">\n";
+  encodingoption += "  <option value=\"w\"";
+  if (arg_w == "w") encodingoption += " selected";
+  encodingoption += ">Western (ISO-8859-1)\n";
+  encodingoption += "  <option value=\"g\"";
+  if (arg_w == "g") encodingoption += " selected";
+  encodingoption += ">Simplified Chinese (GB2312)\n";
+  encodingoption += "  <option value=\"8\"";
+  if (arg_w == "8") encodingoption += " selected";
+  encodingoption += ">Unicode (UTF-8)\n";
+  encodingoption += "</select>\n";
+
+  disp.setmacro ("encodingoption", "preferences", encodingoption);
 
   // the maxdocoption

trunk/gsdl/src/library/libinterface.h

@@ -110 +110 @@
   inconvertclass asciiinconvert;
   utf8inconvertclass utf8inconvert;
-  gbinconvertclass *gbinconvert;
+  mapinconvertclass gbinconvert;
   outconvertclass asciioutconvert;
   utf8outconvertclass utf8outconvert;
-  gboutconvertclass *gboutconvert;
+  mapoutconvertclass gboutconvert;
 
 
@@ -130 +130 @@
   virtual void add_default_args (cgiargsclass &args);
   virtual void check_args (cgiargsclass &args);
+  virtual void args_tounicode (cgiargsclass &args, inconvertclass &inconvert);
 
-  virtual void define_general_macros (cgiargsclass &args, ostream &logout);
-  virtual void prepare_page (cgiargsclass &args, ostream &logout);
+  virtual void define_general_macros (cgiargsclass &args, outconvertclass &outconvert, 
+                      ostream &logout);
+  virtual void prepare_page (cgiargsclass &args, outconvertclass &outconvert,
+                 ostream &logout);
 
 

trunk/gsdl/src/library/mgq.c

@@ -151 +151 @@
               int (*sender)(char *,int,int,float,void *), void *ptr)
 {
-  int i;  char *word;
+  int i;
   for (i = 0; i < qtl->num; i++)
     if (sender != NULL) {
-      word = word2str(qtl->QTE[i].Term);
-      (* sender)(word, strlen(word), qtl->QTE[i].Count, (float)0.0, ptr);
+      /*      word = word2str(qtl->QTE[i].Term); 
+          (* sender)(word, strlen(word), qtl->QTE[i].Count, (float)0.0, ptr); */
+      (* sender)(qtl->QTE[i].Term+1, qtl->QTE[i].Term[0], 
+         qtl->QTE[i].Count, (float)0.0, ptr);
     }
 }
@@ -164 +166 @@
 {
   int i = 0;
-  char *word;
-
   if (sender == NULL) return;
   for (i = 0; i < qtl->num; i++)
     {
-      word = word2str(qtl->TE[i].Word);
-      (* sender)(word, strlen(word), qtl->TE[i].Count, (float)0.0, ptr);
+      /*      word = word2str(qtl->TE[i].Word);
+          (* sender)(word, strlen(word), qtl->TE[i].Count, (float)0.0, ptr);*/
+      (* sender)(qtl->TE[i].Word+1, qtl->TE[i].Word[0], 
+         qtl->TE[i].Count, (float)0.0, ptr);
     }
 }

trunk/gsdl/src/library/mgsearch.cpp

@@ -73 +73 @@
   queryresultsclass *queryresults = (queryresultsclass *)info;
 
+  text_t term;
+  term.setcarr(Word, ULen);
   termfreqclass termfreq;
-  termfreq.termstr.setcarr(Word, ULen);
+  termfreq.termstr = to_uni(term);
   termfreq.termfreq = Freq;
   queryresults->terms.push_back(termfreq);
@@ -85 +87 @@
           float Weight, void *info) {
 
-  // convert term from utf8 to unicode
   text_t term;
-  utf8inconvertclass inconvert;
-  convertclass::status_t status;
-  inconvert.reset ();
-  inconvert.setinput (Word, ULen);
-  inconvert.convert (term, status);
-
+  term.setcarr(Word, ULen);
   queryresultsclass *queryresults = (queryresultsclass *)info;
-  queryresults->termvariants.push_back(term);
+  queryresults->termvariants.push_back(to_uni(term));
 
   return 0;
@@ -255 +251 @@
 
   // quotedquery will be deleted on the next call to this function
-  quotedquery = ttquotedquery.getcstr ();
-  char *querystring = ttquerystring.getcstr();
+  quotedquery = to_utf8(ttquotedquery).getcstr ();
+  char *querystring = to_utf8(ttquerystring).getcstr();
   
   // submit the query
@@ -318 +314 @@
 
 
-void mgsearchclass::filterquery (text_t &ttquerystring)
-{
-
+void mgsearchclass::filterquery (text_t &ttquerystring) {
   text_t::iterator ithere = ttquerystring.begin ();
   text_t::iterator itend = ttquerystring.end ();
-  unsigned short c;
-
-  // remove all non alphanumeric characters below 127
-  while (ithere != itend)
-    {
-      c = *ithere;
-
-      //      if ((c <= 127) && !((c >= '0' && c <= '9') ||
-      //              (c >= 'A' && c <= 'Z') ||
-      //              (c >= 'a' && c <= 'z')))
-      if (!(((c >= 65) && (c <= 90)) ||
-        ((c >= 97) && (c <= 122)) ||
-        ((c >= 192) && (c <= 214)) ||
-        ((c >= 216) && (c <= 246)) ||
-        ((c >= 248) && (c <= 255)) ||
-        ((c >= '0') && (c <= '9')) ||
-        (c == 176)))
-    (*ithere) = ' ';
-      
-      ithere++;
-    }
+  
+  // remove all non alphanumeric characters
+  while (ithere != itend) {
+    if (!is_unicode_letdig(*ithere)) (*ithere) = ' ';
+    ithere++;
+  }
 }