Changeset 2693 for trunk/gsdl/src

Timestamp:

2001-08-10T12:20:23+12:00 (23 years ago)

Author:

kjm18

Message:

changed this to parse terms in the same way that the indexer does. Terms with
more than four digits are broken up, and terms that start with digits
but have letters in them are now treated as terms rather than as integers
eg 12345678 is split into 1234 5678 rather than staying as 12345678
100F stays as 100F rather than being split into 100 F

File:

• trunk/gsdl/src/mgpp/text/GSDLQueryLex.cpp (modified) (3 diffs)

trunk/gsdl/src/mgpp/text/GSDLQueryLex.cpp

@@ -57 +57 @@
 
   // read in number part
+  int numeric=0;
   el.num = 0;
   el.lexType = IntegerE;
-  while (c >= '0' && c <= '9') {
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
     el.num = el.num*10 + c - '0';
     AddNChar (here, el.text, charLen);
@@ -70 +72 @@
 }
 
+static bool ParsePotentialInteger(UCArray::const_iterator &here,
+                  UCArray::const_iterator end,
+                  LexEl &el) {
+  el.Clear();
+
+  // this version of end is used in unitool
+  UCArray::const_iterator endMinus1 = end-1;
+
+  int charLen=0;
+  int length=0;
+  unsigned short c; // one character lookahead
+  charLen = parse_utf8_char (here, endMinus1, &c);
+
+  // read in number part
+  int numeric=0;
+  el.num = 0;
+  el.lexType = IntegerE;
+  /* stop integers at 4 digits */
+  while (c >= '0' && c <= '9'&& ++numeric<=MAXNUMERIC) {
+    el.num = el.num*10 + c - '0';
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+  // check the next character -if it is a letter, then have a term, not an integer
+  if (!is_unicode_letter(c)) {
+    // this was just an integer
+    return (!el.text.empty());
+  }
+  // else its a term
+  el.lexType = TermE;
+  el.num = 0;
+  /* this bit taken from ParseIndexWord in words.h*/
+  while (length+charLen<=MAXSTEMLEN && charLen > 0 &&
+     (is_unicode_letter(c) || (is_unicode_digit(c) &&
+                   ++numeric <= MAXNUMERIC))) {
+    AddNChar (here, el.text, charLen);
+    length += charLen;
+    charLen = parse_utf8_char (here, endMinus1, &c);
+  }
+
+  return (!el.text.empty());
+}
 static bool ParseTerm (UCArray::const_iterator &here,
                UCArray::const_iterator end,
@@ -165 +210 @@
     return true;
     
-  } else if (c == '+' || c == '-' ||
-         (c >= '0' && c <= '9')) {
-    return ParseInteger (here, end, el);
+  } else if (c == '+' || c == '-' ) {
+    return  ParseInteger (here, end, el);
+  }
+
+  else if (c >= '0' && c <= '9') {
+    return ParsePotentialInteger (here, end, el);
   }