Changeset 26163 for main

Timestamp:

2012-09-11T11:34:31+12:00 (12 years ago)

Author:

kjdon

Message:

added some comments to make clear whether max_output_length includes the space for the length in position zero or not (not.) And fixed up an off-by-one error to do with the same thing

File:

• main/trunk/greenstone2/common-src/indexers/mgpp/lib/unitool.cpp (modified) (8 diffs)

main/trunk/greenstone2/common-src/indexers/mgpp/lib/unitool.cpp

@@ -1 +1 @@
 #include "unitool.h"
 #include "sysfuncs.h"
-
 
 /* unicode_cat_data is a combination of the letter, digit, and
@@ -1300 +1299 @@
  * unicode equivalent in *value. The length of the utf-8 character
  * is returned */
+/* NOTE:!!!! end points to the last valid char, NOT past the end
+ */
 int parse_utf8_char(const unsigned char *here,
             const unsigned char *end,
@@ -1345 +1346 @@
  * The length of the encoding is returned. If the string was not
  * long enough to encode the character 0 is returned. */
+/* NOTE:!!!! end points to the last valid char, NOT past the end
+ */
 int output_utf8_char (unsigned short value,
               unsigned char *here,
               unsigned char *end) {
   int len = (int)(end - here + 1); /* end is the last character */
+
   if (value < 0x80) {
     /* will be encoded in one byte */
@@ -1698 +1702 @@
 
 
-/* converts a utf-8 word (string with length stored in the first byte
+/* converts a utf-8 word (string with length stored in the first byte)
  * to a Unicode array. To handle all situations the output buffer should
  * be 256 unsigned shorts long. The output will also have the length as
- * the first entry. */
+ * the first entry. 
+ * NOTE! max_output_length is the maximum length of the output string, ie 
+ * one less than the length of the output array, ie should be <=255 if output
+ * buffer is 256 long
+ */
 unsigned short *utf8_word_to_unicode (const unsigned char *input,
                       unsigned short *output,
@@ -1707 +1715 @@
   int inlen = *input;
   const unsigned char *inhere = input+1;
-  const unsigned char *inend = &inhere[inlen-1];
+  const unsigned char *inend = &inhere[inlen-1]; // points to the last valid char, not past the end.
   unsigned short c;
   int clen;
@@ -1714 +1722 @@
   unsigned short *outhere = output+1;
   
-  while (inhere <= inend && outlen < max_output_length-1) {
+  while (inhere <= inend && outlen < max_output_length) {
     /* decode the character */
     clen = parse_utf8_char (inhere, inend, &c);
@@ -1735 +1743 @@
  * entry) to a utf8 encoded word output (with the length stored in
  * the first byte. Only 255 bytes (not characters) can be stored 
- * in the output. */
+ * in the output. 
+ * Note:!! max_output_length is the maximum length of the output string, ie 
+ * one less than the length of the output array, ie should be <=255 if output
+ * buffer is 256 long
+*/
 unsigned char *unicode_to_utf8_word (const unsigned short *input,
                      unsigned char *output,
@@ -1746 +1758 @@
   int outlen = 0;
   unsigned char *outhere = output+1;
-  unsigned char *outend = output+max_output_length;
+  unsigned char *outend = output+max_output_length; // points to last valid char, not past the end
 
   while (in_i < inlen) {