Changeset 5449 for trunk/indexers/mgpp

Timestamp:

2003-09-04T12:44:52+12:00 (21 years ago)

Author:

kjdon

Message:

added a new token type to the Lex module - UnknownE.
this will match any unrecognisable char - can now just skip over unknown chars rather than spitting the dummy and stopping the parsing

Location:

trunk/indexers/mgpp/text

Files:

• GSDLQueryLex.cpp (modified) (1 diff)
• GSDLQueryLex.h (modified) (1 diff)
• GSDLQueryParser.cpp (modified) (3 diffs)

trunk/indexers/mgpp/text/GSDLQueryLex.cpp

@@ -224 +224 @@
 
   // assume it is a term of some sort
-  if (!ParseTerm (here, end, el.text)) return false;
+  if (!ParseTerm (here, end, el.text))  {
+    // parse term returns false if it hasn't parsed anything that is a term
+    // here should be the same as it was before 
+    el.lexType = UnknownE;
+    AddNChar (here, el.text, charLen);
+    return true;
+  }
+  //return false;
 
   UCArray AND; SetCStr (AND, "AND");

trunk/indexers/mgpp/text/GSDLQueryLex.h

@@ -29 +29 @@
 enum LexType {NoTypeE, OpenBracketE, CloseBracketE, TermE, AndOpE, OrOpE,
           NotOpE, NearOpE, QuoteE, IntegerE, TermWeightE, StemMethodE,
-          RangeE, AtE, TagE, OpenSquareBracketE, CloseSquareBracketE};
+          RangeE, AtE, TagE, OpenSquareBracketE, CloseSquareBracketE, 
+          UnknownE};
 
 struct LexEl {

trunk/indexers/mgpp/text/GSDLQueryParser.cpp

@@ -247 +247 @@
       if (phrase) phrase=false;
       else phrase=true;
+    } else if (el.lexType == UnknownE) {
+      // just ignore it 
     }
     else {
@@ -286 +288 @@
       break;
       
+    } else if (el.lexType == UnknownE) {
+      // just ignore it
     } else {
       // error 
@@ -431 +435 @@
       curTree = NotAdd (curTree, newTerm); 
       
+    } else if (el.lexType == UnknownE) {
+      // just ignore it 
     } else {