Changeset 12784

Timestamp:

2006-09-20T09:53:39+12:00 (18 years ago)

Author:

kjdon

Message:

modified query parsing for mgpp and lucene. hopefully it will work properly now. lucene no longer uses is_unicode_letdig (which is only valid for unicode 2). advanced form search should now behave the same as plain search, allowing operators etc. lucene no longer does stem/case mods.

File:

• trunk/gsdl/src/recpt/querytools.cpp (modified) (10 diffs)

trunk/gsdl/src/recpt/querytools.cpp

@@ -28 +28 @@
 #include "unitool.h" // for is_unicode_letdig
 
+// sets the ct, qt, qto arguments
 void set_query_type_args(ColInfoResponse_t *cinfo, cgiargsclass &args) {
 
@@ -53 +54 @@
   
   text_tmap::iterator check = cinfo->format.find("SearchTypes");
-  text_t search_types = "plain,form";
-  if(check != cinfo->format.end()){
+  text_t search_types;
+  if(check != cinfo->format.end() && !(*check).second.empty()){
     search_types = (*check).second;
-    if (search_types.empty()) {
-      search_types = "plain,form";
-    }
-  }
+  } else {
+    // assume plain,form
+    if (args["qto"].empty()) args["qto"] = "3";
+    if (args["qt"].empty()) {
+      int arg_qto = args.getintarg("qto");
+      if (arg_qto > 1) {
+    args["qt"] = "1";
+      } else {
+    args["qt"] = "0";
+      }
+    }
+    return;
+  }
+  
   
   if (args["qto"].empty()) {
@@ -73 +84 @@
   
   if (args["qt"].empty()) {
-    bool form_default = false;
     int arg_qto = args.getintarg("qto");
     if (arg_qto == 2 || (arg_qto == 3 && starts_with(search_types, "form"))) {
@@ -85 +95 @@
 // request.filterResultOptions and request.fields (if required) should
 // be set from the calling code
-void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring,
+void set_queryfilter_options (FilterRequest_t &request, 
+                  const text_t &querystring,
                   cgiargsclass &args) {
 
@@ -162 +173 @@
 }
 
-void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring1,
+void set_queryfilter_options (FilterRequest_t &request, 
+                  const text_t &querystring1,
                   const text_t &querystring2, cgiargsclass &args) {
 
@@ -212 +224 @@
 }
 
-void set_more_queryfilter_options (FilterRequest_t &request, cgiargsclass &args) {
+void set_more_queryfilter_options (FilterRequest_t &request, 
+                   cgiargsclass &args) {
 
   OptionValue_t option;
@@ -238 +251 @@
   }
   // lucene
-  else if (indexer_type ==2) {
+  else if (indexer_type == 2) {
     return (character == '?' || character == '*' || character == '~' || 
         character == '^');
@@ -245 +258 @@
 }
 
+// This function removes boolean operators from simple searches, and segments
+// chinese characters if segment=true
 void format_querystring (text_t &querystring, int querymode, bool segment) {
   text_t formattedstring;
 
+  // advanced search, no segmenting, don't need to do anything
   if (querymode == 1 && !segment) return;
   
@@ -290 +306 @@
 
     
+
+// search history tool 
+// also used for form query macros
+text_t escape_quotes(const text_t &querystring) {
+
+  text_t::const_iterator here = querystring.begin();
+  text_t::const_iterator end = querystring.end();
+ 
+  text_t escquery = "";
+  while (here != end) {
+    if (*here != '\'' && *here != '\"' && *here != '\n' && *here != '\r') escquery.push_back(*here);
+    else if (*here == '\n' || *here == '\r') {
+      escquery.push_back(' ');
+    } else {
+      escquery +="\\\\";
+      escquery.push_back(*here);
+    }
+
+    ++here;
+  }
+  return escquery;
+
+}
+
+// Parses the terms into words, and adds #si if necessary
+text_t addstemcase(const text_t &terms, const text_t &stem, const text_t &fold,
+           const int indexer_type) {
+  
+  // the default stem and case are set to 0 if this is being used, so we are only adding on qualifiers if stem or fold is 1.
+  if (stem == "0" && fold == "0") {
+    return;
+  }
+  // this is only for mgpp collections, shouldn't be called for anything else
+  if (indexer_type != 1) {
+    return;
+  }
+  
+  text_t outtext;
+  text_t word;
+
+  text_t::const_iterator here = terms.begin();
+  text_t::const_iterator end = terms.end();
+
+  while (here !=end) {
+
+    if (is_unicode_letdig(*here) || is_special_character(indexer_type, *here)) {
+      // not word boundary
+      word.push_back(*here);
+      ++here;    
+    }
+    else {
+      // found word boundary   
+      if (!word.empty() ) {
+    if (starts_with(word, "NEAR") || starts_with(word, "WITHIN")) {
+      outtext += word;
+      word.clear();
+    }
+    else {
+      word += "#";
+      if (stem == "1") word += "s";
+      if (fold == "1") word += "i";
+      outtext += word;
+      word.clear();
+    }
+      }
+      // this only used in advanced form, so we leave in boolean operators
+      if (*here == '\"' || *here == '&' || *here == '|' || *here == '!' || is_unicode_space(*here)) {
+    outtext.push_back(*here);
+      }
+      ++here;
+    }
+  }
+   
+  // get last word
+  if (!word.empty()) {
+    word += "#";
+    if (stem == "1") word += "s";
+    if (fold == "1") word += "i";
+    word += " ";
+    outtext += word;
+  }
+  return outtext;
+}
+
+
+// some query form parsing functions for use with mgpp & lucene
+
+void parse_reg_query_form(text_t &querystring, cgiargsclass &args, bool segment)
+{
+  querystring.clear();
+
+  int argct = args.getintarg("ct");
+  int argt = args.getintarg("t");// t=0 -and, t=1 - or
+  int argb = args.getintarg("b");
+    
+  text_t combine;
+
+  // lucene uses global combine, so only need this for mgpp
+  if (argct==1) {
+    if (argt == 0) combine = "&";
+    else combine = "|";
+  }
+  
+  text_t field = args["fqf"];
+  if (field.empty()) return; // no query
+  text_tarray fields;
+  splitchar(field.begin(), field.end(), ',', fields); 
+  
+  text_t value = args["fqv"];
+  if (value.empty()) return; // somethings wrong
+  text_tarray values;
+  splitchar(value.begin(), value.end(), ',', values);
+
+
+  for (int i=0; i< values.size(); ++i) {
+    if (!values[i].empty()) {
+      text_t this_value = values[i];
+      // remove operators for simple search, segments text if necessary
+      format_querystring(this_value, argb, segment); 
+      // add tag info for this field (and other processing)
+      format_field_info(this_value, fields[i], argct, argt, argb);
+      // add into query string
+      if (argct == 2) {
+    // lucene
+    // we don't worry about AND/OR, cos this is done by defaultcombineoperator
+    querystring += this_value+" ";
+      } else {
+    // mgpp
+    if (!querystring.empty()) {
+      querystring += " "+ combine+ " ";
+    }
+    querystring += this_value;
+      }
+    }
+  }
+}
+
+
+void parse_adv_query_form(text_t &querystring, cgiargsclass &args, bool segment){
+  querystring.clear();
+
+  const int argct = args.getintarg("ct");
+  int argt = 0;// arg t is either not used (lucene) or used for natural/ranked (mgpp), so we set it to 0 = AND, by default
+  int argb = args.getintarg("b");
+  text_t combine;
+  if (argct==1) {
+    combine = "&";
+  }
+  else { // lucene
+    combine = "AND";
+  }
+
+  text_t field = args["fqf"];
+  if (field.empty()) return; // no query
+  text_tarray fields;
+  splitchar(field.begin(), field.end(), ',', fields); 
+  
+  text_t value = args["fqv"];
+  if (value.empty()) return; // somethings wrong
+  text_tarray values;
+  splitchar(value.begin(), value.end(), ',', values);
+
+  text_t comb = args["fqc"];
+  if (comb.empty()) return; //somethings wrong
+  text_tarray combs;
+  splitchar(comb.begin(), comb.end(), ',', combs);
+
+  text_tarray stems;
+  text_tarray folds;
+  if (argct == 1) {// mgpp - lucene doesn't do stem/case
+    text_t stem = args["fqs"];
+    if (stem.empty()) return; // somethings wrong
+    splitchar(stem.begin(), stem.end(), ',', stems);
+    
+    text_t fold = args["fqk"];
+    if (fold.empty()) return; // somethings wrong
+    splitchar(fold.begin(), fold.end(), ',', folds);
+  }
+  
+  for(int i=0; i< values.size(); ++i) {
+    if (!values[i].empty()) {
+      if (i!=0) {
+    if (argct==1) {
+      if (combs[i-1]=="and") combine = "&";
+      else if (combs[i-1]=="or")combine = "|";
+      else if (combs[i-1]=="not")combine = "!";
+    }
+    else { // lucene
+      if (combs[i-1]=="and") combine = "AND";
+      else if (combs[i-1]=="or")combine = "OR";
+      else if (combs[i-1]=="not")combine = "NOT";
+    }
+      }
+      text_t this_value = values[i];
+      // remove operators for simple search, segments text if necessary
+      format_querystring(this_value, argb, segment); 
+      if (argct == 1) { // mgpp only
+    this_value = addstemcase(this_value, stems[i], folds[i], argct);
+      }
+      // add tag info for this field (and other processing)
+      format_field_info(this_value, fields[i], argct, argt, argb);
+      // add into query string
+      if (!querystring.empty()) {
+    querystring += " "+ combine+ " ";
+      }
+      querystring += this_value;
+      
+    }
+  }
+}
+
+// Extended addqueryelem for Human Info project
+void addqueryelem_ex(text_t &querystring, const text_t &tag, 
+             const text_t &terms, const text_t &stem, 
+             const text_t &fold,
+             const text_t& combine, const text_t& word_combine) {
+
+  if (!querystring.empty()) { // have to put and/or
+    querystring += " " + combine + " ";
+  }
+  text_t outtext; outtext.reserve(512);
+  text_t word; word.reserve(100);
+  //unsigned short c;                                                            
+  text_t::const_iterator here = terms.begin();
+  text_t::const_iterator end = terms.end();
+  bool inquote = false, firstword = true;
+
+  text_t word2; word2.reserve(256);
+    
+  while (here !=end) {
+    if (is_unicode_space(*here)) {
+      if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
+      else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
+      else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
+      else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
+      else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
+      if (inquote) {
+    word2.push_back(*here);
+      }
+      word.append(word2); word2.clear();
+            
+      if (!inquote && !word.empty() ) {
+    // found word boundary   
+                
+    if (stem == "1" || fold =="1") {
+      word += "#";
+      if (stem == "1") word += "s";
+      //else word += "u";
+                    
+      if (fold == "1") word += "i";
+      //else word += "c";
+    }
+    if (firstword) {
+      firstword = false;
+    } else {
+      outtext += " " + word_combine + " ";
+    }
+    outtext += "[" + word + "]:"+tag;
+    word.clear();
+      }
+      ++here;
+    } else if (*here == '\"') {
+      word2.push_back(*here);
+      inquote = !inquote;
+      ++here;
+    } else {
+      // not word boundary
+      word2.push_back(*here);
+      ++here;    
+    }
+  }
+    
+  // get last word
+  if (!word2.empty()) {
+    if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
+    else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
+    else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
+    else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
+    else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
+    word.append(word2); word2.clear();
+        
+    if (stem == "1"|| fold == "1") {
+      word += "#";
+      if (stem == "1") word += "s";
+      //else word += "u";
+            
+      if (fold == "1") word += "i";
+      //else word += "c";
+    }
+    if (!outtext.empty()) outtext += " " + word_combine + " ";
+    outtext += "[" + word + "]:"+tag;
+  }
+  querystring += "(" + outtext + ")";
+}
+
+void add_field_info(text_t &querystring, const text_t &tag, int type) {
+
+  if (tag == "" || tag == "ZZ") return; // do nothing
+  if (type == 1) { //mgpp
+    querystring = "["+querystring+"]:"+tag;
+  } else if (type == 2) { // lucene
+    querystring = tag+":("+querystring+")";
+  }
+    
+}
+
+
+void format_field_info_lucene(text_t &querystring, text_t tag, int argt, int argb) {
+  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
+  int type = 2; //lucene
+
+  if (argb==0) { // simple
+    // there will be no & or | as they should have already been removed
+    // just tag the entire thing
+    if (tag != "") {
+      add_field_info(querystring, tag, type);
+    }
+    return;
+  }
+
+  // need to replace & with &&, | with ||
+  text_t::const_iterator here = querystring.begin();
+  text_t::const_iterator end = querystring.end();
+
+  text_t finalquery = "";
+  while (here != end) {
+    if (*here ==  '&') {
+      finalquery.push_back('&');
+      finalquery.push_back('&');
+      while (*(here+1) == '&') {
+    ++here;
+      }
+    } 
+    else if (*here == '|') {
+      finalquery.push_back('|');
+      finalquery.push_back('|');
+      while (*(here+1) == '|') {
+    ++here;
+      }
+    } 
+    else {
+      finalquery.push_back(*here);
+    }
+    ++here;
+  }
+  querystring = finalquery;
+  add_field_info(querystring, tag, type);
+}
+
+
+void format_field_info_mgpp(text_t &querystring, text_t tag, int argt, int argb) {
+
+  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
+  if (tag == "" && argb == 1) {
+    return; // no field specifier, advanced mode, the query stays as written
+  }
+
+  int type = 1; // mgpp
+
+  bool simple_and = (argb==0 && argt==0); 
+  text_t finalquery = "";
+  text_t fieldpart ="";
+  text_t queryelem = "";
+  bool in_phrase = false;
+  bool in_field = false;
+
+  text_t::const_iterator here = querystring.begin();
+  text_t::const_iterator end = querystring.end();
+  while (here != end) {
+    if (is_unicode_letdig(*here)  || *here == '&' || is_special_character(type, *here)) {
+      queryelem.push_back(*here);
+    }
+    else if (*here == '|') {
+      in_field = false;
+    }
+    else if (*here == '!' || *here == '(' || *here == ')') {
+      if (!in_phrase) { // ignore these if in_phrase
+    // output field, then output operator
+    in_field = false;
+    if (!queryelem.empty()) {
+      if (!simple_and && !fieldpart.empty()) {
+        add_field_info(fieldpart, tag, type);
+        finalquery += fieldpart;
+        finalquery.push_back(' ');
+        fieldpart.clear();
+      }
+      fieldpart += queryelem;
+    }
+    if (!fieldpart.empty()) {
+      add_field_info(fieldpart, tag, type);
+      finalquery += fieldpart;
+      finalquery.push_back(' ');
+    }
+    fieldpart.clear();
+    queryelem.clear();
+    finalquery.push_back(*here);
+    finalquery.push_back(' ');
+      }
+    }
+    else if (*here == '"') {
+      queryelem.push_back(*here);
+      if (in_phrase == false) in_phrase = true;
+      else {
+    in_phrase = false;
+      }
+    }
+
+    // Found word boundary, in a phrase
+    else if (in_phrase) {
+      queryelem.push_back(*here);
+    }
+    // Found a word boundary
+    else {
+      if (!queryelem.empty()) {
+    if (queryelem == "&") {
+      in_field = true;
+      queryelem.clear();
+    }
+    else if (starts_with(queryelem, "NEAR") || starts_with(queryelem, "WITHIN")) {
+      
+      if (argb==1) {
+        // simple search, these not allowed
+        in_field = true;
+        fieldpart += queryelem;
+        fieldpart.push_back(' ');
+      }
+      queryelem.clear();
+      
+    } 
+    else {
+      if (!simple_and && !in_field) {
+        if (!fieldpart.empty()) {
+          add_field_info(fieldpart, tag, type);
+          finalquery += fieldpart;
+          finalquery.push_back(' ');
+          fieldpart.clear();
+        } 
+      }
+      
+      fieldpart += queryelem;
+      fieldpart.push_back(' ');
+      queryelem.clear();
+    }
+      }
+    }
+    ++here;
+  }
+  // at the end
+  if (!queryelem.empty()) {
+    if (!simple_and && !in_field && !fieldpart.empty()) {
+      add_field_info(fieldpart, tag, type);
+      finalquery += fieldpart;
+      finalquery.push_back(' ');
+      fieldpart.clear();
+    }
+    fieldpart += queryelem;
+  }
+  if (!fieldpart.empty()) {
+    add_field_info(fieldpart, tag, type);
+    finalquery += fieldpart;
+    fieldpart.clear();
+    finalquery.push_back(' ');
+  }
+
+  querystring  = finalquery;
+}
+
+
+void format_field_info(text_t &querystring, text_t tag, int argct, int argt, int argb) {  
+  if (argct == 1) {
+    format_field_info_mgpp(querystring, tag, argt, argb);
+  } else if (argct == 2) {
+    format_field_info_lucene(querystring, tag, argt, argb);
+  }
+}
+
+void mgpp_adddateelem(text_t& querystring, const int date)
+{
+  querystring.appendcstr(" [");
+  if(date<0) {
+      querystring.appendcstr("bc");
+      querystring.appendint((date*-1));
+  }
+  else {
+    querystring.appendint(date);
+  }
+  querystring.appendcstr("]:CV");
+}
+
+void lucene_adddateelem(text_t& querystring, const int date)
+{
+  querystring.appendcstr(" CV:(");
+  if(date<0) {
+      querystring.appendcstr("bc");
+      querystring.appendint((date*-1));
+  }
+  else {
+    querystring.appendint(date);
+  }
+  querystring.appendcstr(")");
+}
+
+
 void add_dates(text_t &querystring, int startdate, int enddate, 
            int startbc, int endbc, int ct)
@@ -352 +871 @@
 
 }
-
-// search history tool 
-// also used for form query macros
-text_t escape_quotes(const text_t &querystring) {
-
-  text_t::const_iterator here = querystring.begin();
-  text_t::const_iterator end = querystring.end();
- 
-  text_t escquery = "";
-  while (here != end) {
-    if (*here != '\'' && *here != '\"' && *here != '\n' && *here != '\r') escquery.push_back(*here);
-    else if (*here == '\n' || *here == '\r') {
-      escquery.push_back(' ');
-    } else {
-      escquery +="\\\\";
-      escquery.push_back(*here);
-    }
-
-    ++here;
-  }
-  return escquery;
-
-}
-
-// some query form parsing functions for use with mgpp & lucene
-
-void parse_reg_query_form(text_t &querystring, cgiargsclass &args)
-{
-  querystring.clear();
-
-  const int ct = args.getintarg("ct");
-  int argt = args.getintarg("t");// t=0 -and, t=1 - or
-
-  text_t combine;
-  if (ct==1) {
-    if (argt == 0) combine = "&";
-    else combine = "|";
-  }
-  else { // lucene
-    if (argt == 0) combine = "AND";
-    else combine = "OR";
-  }
-  
-  text_t field = args["fqf"];
-  if (field.empty()) return; // no query
-  text_tarray fields;
-  splitchar(field.begin(), field.end(), ',', fields); 
-  
-  text_t value = args["fqv"];
-  if (value.empty()) return; // somethings wrong
-  text_tarray values;
-  splitchar(value.begin(), value.end(), ',', values);
-
-
-  for (int i=0; i< values.size(); ++i) {
-    if (!values[i].empty()) {
-      if (ct == 1) {
-    mgpp_addqueryelem(querystring, fields[i], values[i], combine);
-      }
-      else { // lucene
-    lucene_addqueryelem(querystring, fields[i], values[i], combine);
-      }
-    }
-  }
-  
-}
-
-
-void parse_adv_query_form(text_t &querystring, cgiargsclass &args){
-
-  querystring.clear();
-
-  const int ct = args.getintarg("ct");
-  text_t combine;
-  if (ct==1) {
-    combine = "&";
-  }
-  else { // lucene
-    combine = "AND";
-  }
-
-  text_t field = args["fqf"];
-  if (field.empty()) return; // no query
-  text_tarray fields;
-  splitchar(field.begin(), field.end(), ',', fields); 
-  
-  text_t value = args["fqv"];
-  if (value.empty()) return; // somethings wrong
-  text_tarray values;
-  splitchar(value.begin(), value.end(), ',', values);
-
-  text_t stem = args["fqs"];
-  if (stem.empty()) return; // somethings wrong
-  text_tarray stems;
-  splitchar(stem.begin(), stem.end(), ',', stems);
-
-  text_t fold = args["fqk"];
-  if (fold.empty()) return; // somethings wrong
-  text_tarray folds;
-  splitchar(fold.begin(), fold.end(), ',', folds);
-
-  text_t comb = args["fqc"];
-  if (comb.empty()) return; //somethings wrong
-  text_tarray combs;
-  splitchar(comb.begin(), comb.end(), ',', combs);
-  
-  for(int i=0; i< values.size(); ++i) {
-    if (!values[i].empty()) {
-      if (i!=0) {
-    if (ct==1) {
-      if (combs[i-1]=="and") combine = "&";
-      else if (combs[i-1]=="or")combine = "|";
-      else if (combs[i-1]=="not")combine = "!";
-    }
-    else { // lucene
-      if (combs[i-1]=="and") combine = "AND";
-      else if (combs[i-1]=="or")combine = "OR";
-      else if (combs[i-1]=="not")combine = "NOT";
-    }
-      }
-      text_t term = addstemcase(values[i], stems[i], folds[i], ct);
-      mgpp_addqueryelem(querystring, fields[i], term, combine);
-    }
-    
-  }
-}
-
-text_t addstemcase(const text_t &terms, const text_t &stem, const text_t &fold,
-           const int indexer_type) {
-  
-  text_t outtext;
-  text_t word;
-  //unsigned short c;                                                            
-  text_t::const_iterator here = terms.begin();
-  text_t::const_iterator end = terms.end();
-
-  while (here !=end) {
-
-    if (is_unicode_letdig(*here) || is_special_character(indexer_type, *here)) {
-      // not word boundary
-      word.push_back(*here);
-      ++here;    
-    }
-    else {
-      // found word boundary   
-      if (!word.empty() ) {
-    if (stem == "1" || fold =="1") {
-      word += "#";
-      if (stem == "1") word += "s";
-      //else word += "u";
-      
-      if (fold == "1") word += "i";
-      //else word += "c";
-    }
-    
-    word += " ";
-    outtext += word;
-    word.clear();
-      }
-      if (*here == '\"') {
-    outtext.push_back(*here);
-      }
-      ++here;
-    }
-  }
-   
-  // get last word
-  if (!word.empty()) {
-    if (stem == "1"|| fold == "1") {
-      word += "#";
-      if (stem == "1") word += "s";
-      //else word += "u";
-      
-      if (fold == "1") word += "i";
-      //else word += "c";
-    }
-    word += " ";
-    outtext += word;
-  }
-  return outtext;
-}
-
-
-void mgpp_adddateelem(text_t& querystring, const int date)
-{
-  querystring.appendcstr(" [");
-  if(date<0) {
-      querystring.appendcstr("bc");
-      querystring.appendint((date*-1));
-  }
-  else {
-    querystring.appendint(date);
-  }
-  querystring.appendcstr("]:CV");
-}
-
-void lucene_adddateelem(text_t& querystring, const int date)
-{
-  querystring.appendcstr(" CV:(");
-  if(date<0) {
-      querystring.appendcstr("bc");
-      querystring.appendint((date*-1));
-  }
-  else {
-    querystring.appendint(date);
-  }
-  querystring.appendcstr(")");
-}
-
-
-void mgpp_addqueryelem(text_t &querystring, text_t &tag, 
-          text_t &query, text_t &combine) {
-  if (!querystring.empty()) { // have to put and/or
-    querystring += " " + combine + " ";
- 
-  }
-  if (tag=="ZZ" || tag=="") { // just add onto querystring
-       querystring +=  query;
-  }
-  else {
-    querystring += "["+query+"]:"+tag;
-  }
-
-}
-
-void lucene_addqueryelem(text_t &querystring, text_t &tag, 
-          text_t &query, text_t &combine) {
-  if (!querystring.empty()) { // have to put and/or
-    querystring += " " + combine + " ";
- 
-  }
-  if (tag=="ZZ" || tag=="") { // just add onto querystring
-       querystring +=  query;
-  }
-  else {
-    querystring += tag+":("+query+")";
-  }
-}
-
-
-void addqueryelem_ex(text_t &querystring, const text_t &tag, 
-             const text_t &terms, const text_t &stem, const text_t &fold,
-             const text_t& combine, const text_t& word_combine) {
-  if (!querystring.empty()) { // have to put and/or
-    querystring += " " + combine + " ";
-  }
-  text_t outtext; outtext.reserve(512);
-  text_t word; word.reserve(100);
-  //unsigned short c;                                                            
-  text_t::const_iterator here = terms.begin();
-  text_t::const_iterator end = terms.end();
-  bool inquote = false, firstword = true;
-
-  text_t word2; word2.reserve(256);
-    
-  while (here !=end) {
-    if (is_unicode_space(*here)) {
-      if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
-      else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
-      else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
-      else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
-      else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
-      if (inquote) {
-    word2.push_back(*here);
-      }
-      word.append(word2); word2.clear();
-            
-      if (!inquote && !word.empty() ) {
-                // found word boundary   
-                
-    if (stem == "1" || fold =="1") {
-      word += "#";
-      if (stem == "1") word += "s";
-      //else word += "u";
-                    
-      if (fold == "1") word += "i";
-      //else word += "c";
-    }
-    if (firstword) {
-      firstword = false;
-    } else {
-      outtext += " " + word_combine + " ";
-    }
-    outtext += "[" + word + "]:"+tag;
-    word.clear();
-      }
-      ++here;
-    } else if (*here == '\"') {
-      word2.push_back(*here);
-      inquote = !inquote;
-      ++here;
-    } else {
-      // not word boundary
-      word2.push_back(*here);
-      ++here;    
-    }
-  }
-    
-  // get last word
-  if (!word2.empty()) {
-    if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
-    else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
-    else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
-    else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
-    else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
-    word.append(word2); word2.clear();
-        
-    if (stem == "1"|| fold == "1") {
-      word += "#";
-      if (stem == "1") word += "s";
-      //else word += "u";
-            
-      if (fold == "1") word += "i";
-      //else word += "c";
-    }
-    if (!outtext.empty()) outtext += " " + word_combine + " ";
-    outtext += "[" + word + "]:"+tag;
-  }
-  querystring += "(" + outtext + ")";
-}
-
-
-void add_field_info(text_t &querystring, const text_t &tag, int type) {
-
-  if (tag == "") return; // do nothing
-  if (type == 1) { //mgpp
-    querystring = "["+querystring+"]:"+tag;
-  } else if (type == 2) { // lucene
-    querystring = tag+":("+querystring+")";
-  }
-    
-}
-
-
-void format_field_info_lucene(text_t &querystring, cgiargsclass &args) {
-  text_t tag = args["fqf"];
-  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
-  int type = 2; //lucene
-  int argt = args.getintarg("t");// t=0 -and, t=1 - or
-  int argb = args.getintarg("b"); // b=0 simple, b=1 advanced
-
-  // lucene simple OR - the string stays as is, but may need field tag
-  if (argb==0 && argt == 1) {
-    // just tag the entire thing
-    if (tag != "") {
-      add_field_info(querystring, tag, type);
-    }
-    return;
-  }
-  bool in_phrase = false;
-  
-  text_t queryelem = "";
-  text_t finalquery = "";
-  
-  // only add in + for simple AND search
-  text_t combine = ((argb==0)? "+" : "");
-
-  // for lucene, we need to change & to && and | to || if advanced search
-  // we need to tag the entire string, if we have a field
-  // if we are simple and search, then we put && in between words
-  
-  text_t::const_iterator here = querystring.begin();
-  text_t::const_iterator end = querystring.end();
-  while (here != end) {
-    if (is_unicode_letdig(*here) || is_special_character(type, *here)) {
-      queryelem.push_back(*here);
-    }
-
-    // Detect phrase starts/finishes
-    else if (*here == '"') {
-      queryelem.push_back(*here);
-      if (in_phrase == false) in_phrase = true;
-      else {
-    finalquery += combine + queryelem;
-    queryelem.clear();
-    in_phrase = false;
-      }
-    }
-
-    // Found word boundary, in a phrase
-    else if (in_phrase) {
-      queryelem.push_back(*here);
-    }
-    // Word boundary, but not in a phrase
-    else {
-      if (*here == '&') {
-    queryelem.push_back('&');
-    queryelem.push_back('&');
-      } else if (*here == '|') {
-    queryelem.push_back('|');
-    queryelem.push_back('|');
-      } else {
-    if (!queryelem.empty()) {
-      finalquery += combine + queryelem;
-      queryelem.clear();
-    }
-    finalquery.push_back(*here);
-      }
-    }
-
-    ++here;
-  }
-
-  // Get last element
-  if (!queryelem.empty()) {
-    finalquery += combine + queryelem;
-  }
-
-  add_field_info(finalquery, tag, type);
-  querystring = finalquery;
-}
-
-void format_field_info_mgpp(text_t &querystring, cgiargsclass &args) {
-  text_t tag = args["fqf"];
-  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
- 
-  int argt = args.getintarg("t");// t=0 -and, t=1 - or
-  int argb = args.getintarg("b"); // b=0 simple, b=1 advanced
-
-  if (tag == "" && argb ==1) {
-    return; // no field specifier, advanced mode, the query stays as written
-  }
-
-  int type = 1; // mgpp
-
-  bool simple_and = (argb==0 && argt==0); 
-  text_t finalquery = "";
-  text_t fieldpart ="";
-  text_t queryelem = "";
-  bool in_phrase = false;
-  bool in_field = false;
-
-  text_t::const_iterator here = querystring.begin();
-  text_t::const_iterator end = querystring.end();
-  while (here != end) {
-    if (is_unicode_letdig(*here)  || *here == '&' || is_special_character(type, *here)) {
-      queryelem.push_back(*here);
-    }
-    else if (*here == '|') {
-      in_field = false;
-    }
-    else if (*here == '!' || *here == '(' || *here == ')') {
-      if (!in_phrase) { // ignore these if in_phrase
-    // output field, then output operator
-    in_field = false;
-    if (!queryelem.empty()) {
-      if (!simple_and && !fieldpart.empty()) {
-        add_field_info(fieldpart, tag, type);
-        finalquery += fieldpart;
-        finalquery.push_back(' ');
-        fieldpart.clear();
-      }
-      fieldpart += queryelem;
-    }
-    if (!fieldpart.empty()) {
-      add_field_info(fieldpart, tag, type);
-      finalquery += fieldpart;
-      finalquery.push_back(' ');
-    }
-    fieldpart.clear();
-    queryelem.clear();
-    finalquery.push_back(*here);
-    finalquery.push_back(' ');
-      }
-    }
-    else if (*here == '"') {
-      queryelem.push_back(*here);
-      if (in_phrase == false) in_phrase = true;
-      else {
-    in_phrase = false;
-      }
-    }
-
-    // Found word boundary, in a phrase
-    else if (in_phrase) {
-      queryelem.push_back(*here);
-    }
-    // Found a word boundary
-    else {
-      if (!queryelem.empty()) {
-    if (queryelem == "&") {
-      in_field = true;
-      queryelem.clear();
-    }
-    else if (starts_with(queryelem, "NEAR") || starts_with(queryelem, "WITHIN")) {
-      
-      if (argb==1) {
-        // simple search, these not allowed
-        in_field = true;
-        fieldpart += queryelem;
-        fieldpart.push_back(' ');
-      }
-      queryelem.clear();
-      
-    } 
-    else {
-      if (!simple_and && !in_field) {
-        if (!fieldpart.empty()) {
-          add_field_info(fieldpart, tag, type);
-          finalquery += fieldpart;
-          finalquery.push_back(' ');
-          fieldpart.clear();
-        } 
-      }
-      
-      fieldpart += queryelem;
-      fieldpart.push_back(' ');
-      queryelem.clear();
-    }
-      }
-    }
-    ++here;
-  }
-  // at the end
-  if (!queryelem.empty()) {
-    if (!simple_and && !in_field && !fieldpart.empty()) {
-      add_field_info(fieldpart, tag, type);
-      finalquery += fieldpart;
-      finalquery.push_back(' ');
-      fieldpart.clear();
-    }
-    fieldpart += queryelem;
-  }
-  if (!fieldpart.empty()) {
-    add_field_info(fieldpart, tag, type);
-    finalquery += fieldpart;
-    fieldpart.clear();
-    finalquery.push_back(' ');
-  }
-
-  querystring  = finalquery;
-  cerr << "final query = "<<finalquery<<endl;
-}
-
-void format_field_info(text_t &querystring, cgiargsclass &args) {
-  int argct = args.getintarg("ct");
-  if (argct == 1) {
-    format_field_info_mgpp(querystring, args);
-  } else if (argct == 2) {
-    format_field_info_lucene(querystring, args);
-  }
-}
-