source: trunk/gsdl/src/recpt/querytools.cpp@ 11765

/**********************************************************************
 *
 * querytools.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

#include "querytools.h"
#include <ctype.h>
#include "unitool.h" // for is_unicode_letdig

// request.filterResultOptions and request.fields (if required) should
// be set from the calling code
void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring,
                  cgiargsclass &args) {

  request.filterName = "QueryFilter";

  OptionValue_t option;

  option.name = "Term";
  option.value = querystring;
  request.filterOptions.push_back (option);

  option.name = "QueryType";
  option.value = (args.getintarg("t")) ? "ranked" : "boolean";
  request.filterOptions.push_back (option);

  option.name = "MatchMode";
  // mgpp in advanced mode, always use some query
  if (args.getintarg("ct") !=0 && args.getintarg("b") == 1) {
    option.value = "some";
  } else {
    option.value = (args.getintarg("t")) ? "some" : "all";
  }
  request.filterOptions.push_back (option);

  option.name = "Casefold";
  option.value = (args.getintarg("k")) ? "true" : "false";
  request.filterOptions.push_back (option);

  option.name = "Stem";
  option.value = (args.getintarg("s")) ? "true" : "false";
  request.filterOptions.push_back (option);

  if (!args["h"].empty()) {
    option.name = "Index";
    option.value = args["h"];
    request.filterOptions.push_back (option);
  }

  if (!args["j"].empty()) {
    option.name = "Subcollection";
    option.value = args["j"];
    request.filterOptions.push_back (option);
  }

  if (!args["n"].empty()) {
    option.name = "Language";
    option.value = args["n"];
    request.filterOptions.push_back (option);
  }
  
  if (!args["g"].empty()) { // granularity for mgpp
    option.name = "Level";
    option.value = args["g"];
    request.filterOptions.push_back (option);
  }

  set_more_queryfilter_options (request, args);
}

void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring1,
                  const text_t &querystring2, cgiargsclass &args) {

  set_queryfilter_options (request, querystring1, args);

  // fill in the second query if needed
  if (!args["cq2"].empty()) {
    OptionValue_t option;

    option.name = "CombineQuery";
    option.value = args["cq2"];
    request.filterOptions.push_back (option);
    
    option.name = "Term";
    option.value = querystring2;
    request.filterOptions.push_back (option);
    
    option.name = "QueryType";
    option.value = (args.getintarg("t")) ? "ranked" : "boolean";
    request.filterOptions.push_back (option);

    option.name = "Casefold";
    option.value = (args.getintarg("k")) ? "true" : "false";
    request.filterOptions.push_back (option);

    option.name = "Stem";
    option.value = (args.getintarg("s")) ? "true" : "false";
    request.filterOptions.push_back (option);

    if (!args["h2"].empty()) {
      option.name = "Index";
      option.value = args["h2"];
      request.filterOptions.push_back (option);
    }

    if (!args["j2"].empty()) {
      option.name = "Subcollection";
      option.value = args["j2"];
      request.filterOptions.push_back (option);
    }

    if (!args["n2"].empty()) {
      option.name = "Language";
      option.value = args["n2"];
      request.filterOptions.push_back (option);
    }
  }
  set_more_queryfilter_options (request, args);
}

void set_more_queryfilter_options (FilterRequest_t &request, cgiargsclass &args) {

  OptionValue_t option;
  int arg_m = args.getintarg("m");
  
  option.name = "Maxdocs";
  option.value = arg_m;
  request.filterOptions.push_back (option);

  //  option.name = "StartResults";
  //  option.value = args["r"];
  //  request.filterOptions.push_back (option);

  //  option.name = "EndResults";
  //  int endresults = args.getintarg("o") + (args.getintarg("r") - 1);
  //  if ((endresults > arg_m) && (arg_m != -1)) endresults = arg_m;
  //  option.value = endresults;
  //  request.filterOptions.push_back (option);
}

void format_querystring (text_t &querystring, int querymode, bool segment) {
  text_t formattedstring;

  if (querymode == 1 && !segment) return;
  
  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();

  // space is used to insert spaces between Chinese
  // characters. No space is needed before the first
  // Chinese character.
  bool space = false;

  // want to remove ()|!& from querystring so boolean queries are just
  // "all the words" queries (unless querymode is advanced)
  while (here != end) {
    if ((querymode == 0) && (*here == '(' || *here == ')' || *here == '|' ||
                 *here == '!' || *here == '&')) {
      formattedstring.push_back(' ');
    } else if (segment) {
      if ((*here >= 0x4e00 && *here <= 0x9fa5) ||
      (*here >= 0xf900 && *here <= 0xfa2d)) {
    // Chinese character
    if (!space) formattedstring.push_back (0x200b); // zero width space
    formattedstring.push_back (*here);
    formattedstring.push_back (0x200b);
    space = true;
      } else {
    
    // non-Chinese character
    formattedstring.push_back (*here);
    space = false;
    
      }
    
    } else {
      formattedstring.push_back (*here);
    }
    ++here;
  }
  querystring = formattedstring;
}


    
void add_dates(text_t &querystring, int startdate, int enddate, 
           int startbc, int endbc, int ct)
{
  if(startdate)
    {
      int querystringis = 0;
      text_t::const_iterator here = querystring.begin();
      text_t::const_iterator end = querystring.end();
      while(here!=end)
    {
      if(!(isspace((*here)))){
        here = end;
        querystringis = 1;
      }
      else
        ++here;
    }
      //converting BCE dates
      if(startbc && startdate > 0)
    {
      startdate *= -1;
    }
      if(endbc && enddate > 0)
    {
      enddate *= -1;
    }
       if(enddate != 0 && enddate<startdate)
    {
      cout<<"enddate too small"<<endl;
      return;
    }
       if(querystringis)
     querystring.appendcstr(" AND");
       if(!enddate)
     {
       if (ct==1) {
         mgpp_adddateelem(querystring,startdate);
       }
       else { // lucene
         lucene_adddateelem(querystring,startdate);
       }
     }
       else{
     int nextdate = startdate;
     querystring.appendcstr(" (");
     while(nextdate<=enddate)
       {
         if(nextdate!=0) {
           if (ct==1) {
         mgpp_adddateelem(querystring,nextdate);
           }
           else { // lucene
         lucene_adddateelem(querystring,nextdate);
           }
         }
         ++nextdate;
       }
     querystring.appendcstr(" )");
       }
    }

}

// search history tool 
// also used for form query macros
text_t escape_quotes(const text_t &querystring) {

  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();
 
  text_t escquery = "";
  while (here != end) {
    if (*here != '\'' && *here != '\"' && *here != '\n' && *here != '\r') escquery.push_back(*here);
    else if (*here == '\n' || *here == '\r') {
      escquery.push_back(' ');
    } else {
      escquery +="\\\\";
      escquery.push_back(*here);
    }

    ++here;
  }
  return escquery;

}

// some query form parsing functions for use with mgpp & lucene

void parse_reg_query_form(text_t &querystring, cgiargsclass &args)
{
  querystring.clear();

  const int ct = args.getintarg("ct");
  int argt = args.getintarg("t");// t=0 -and, t=1 - or

  text_t combine;
  if (ct==1) {
    if (argt == 0) combine = "&";
    else combine = "|";
  }
  else { // lucene
    if (argt == 0) combine = "AND";
    else combine = "OR";
  }
  
  text_t field = args["fqf"];
  if (field.empty()) return; // no query
  text_tarray fields;
  splitchar(field.begin(), field.end(), ',', fields); 
  
  text_t value = args["fqv"];
  if (value.empty()) return; // somethings wrong
  text_tarray values;
  splitchar(value.begin(), value.end(), ',', values);


  for (int i=0; i< values.size(); ++i) {
    if (!values[i].empty()) {
      if (ct == 1) {
    mgpp_addqueryelem(querystring, fields[i], values[i], combine);
      }
      else { // lucene
    lucene_addqueryelem(querystring, fields[i], values[i], combine);
      }
    }
  }
  
}


void parse_adv_query_form(text_t &querystring, cgiargsclass &args){

  querystring.clear();

  const int ct = args.getintarg("ct");
  text_t combine;
  if (ct==1) {
    combine = "&";
  }
  else { // lucene
    combine = "AND";
  }

  text_t field = args["fqf"];
  if (field.empty()) return; // no query
  text_tarray fields;
  splitchar(field.begin(), field.end(), ',', fields); 
  
  text_t value = args["fqv"];
  if (value.empty()) return; // somethings wrong
  text_tarray values;
  splitchar(value.begin(), value.end(), ',', values);

  text_t stem = args["fqs"];
  if (stem.empty()) return; // somethings wrong
  text_tarray stems;
  splitchar(stem.begin(), stem.end(), ',', stems);

  text_t fold = args["fqk"];
  if (fold.empty()) return; // somethings wrong
  text_tarray folds;
  splitchar(fold.begin(), fold.end(), ',', folds);

  text_t comb = args["fqc"];
  if (comb.empty()) return; //somethings wrong
  text_tarray combs;
  splitchar(comb.begin(), comb.end(), ',', combs);
  
  for(int i=0; i< values.size(); ++i) {
    if (!values[i].empty()) {
      if (i!=0) {
    if (ct==1) {
      if (combs[i-1]=="and") combine = "&";
      else if (combs[i-1]=="or")combine = "|";
      else if (combs[i-1]=="not")combine = "!";
    }
    else { // lucene
      if (combs[i-1]=="and") combine = "AND";
      else if (combs[i-1]=="or")combine = "OR";
      else if (combs[i-1]=="not")combine = "NOT";
    }
      }
      text_t term = addstemcase(values[i], stems[i], folds[i]);
      mgpp_addqueryelem(querystring, fields[i], term, combine);
    }
    
  }
}

text_t addstemcase(const text_t &terms, const text_t &stem, const text_t &fold) {
  
  text_t outtext;
  text_t word;
  //unsigned short c;                                                            
  text_t::const_iterator here = terms.begin();
  text_t::const_iterator end = terms.end();

  while (here !=end) {

    if (is_unicode_letdig(*here)) {
      // not word boundary
      word.push_back(*here);
      ++here;    
    }
    else {
      // found word boundary   
      if (!word.empty() ) {
    if (stem == "1" || fold =="1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
      
      if (fold == "1") word += "i";
      //else word += "c";
    }
    
    word += " ";
    outtext += word;
    word.clear();
      }
      if (*here == '\"') {
    outtext.push_back(*here);
      }
      ++here;
    }
  }
   
  // get last word
  if (!word.empty()) {
    if (stem == "1"|| fold == "1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
      
      if (fold == "1") word += "i";
      //else word += "c";
    }
    word += " ";
    outtext += word;
  }
  return outtext;
}


void mgpp_adddateelem(text_t& querystring, const int date)
{
  querystring.appendcstr(" [");
  if(date<0) {
      querystring.appendcstr("bc");
      querystring.appendint((date*-1));
  }
  else {
    querystring.appendint(date);
  }
  querystring.appendcstr("]:CV");
}

void lucene_adddateelem(text_t& querystring, const int date)
{
  querystring.appendcstr(" CV:(");
  if(date<0) {
      querystring.appendcstr("bc");
      querystring.appendint((date*-1));
  }
  else {
    querystring.appendint(date);
  }
  querystring.appendcstr(")");
}


void mgpp_addqueryelem(text_t &querystring, text_t &tag, 
          text_t &query, text_t &combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
 
  }
  if (tag=="ZZ" || tag=="") { // just add onto querystring
       querystring +=  query;
  }
  else {
    querystring += "["+query+"]:"+tag;
  }

}

void lucene_addqueryelem(text_t &querystring, text_t &tag, 
          text_t &query, text_t &combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
 
  }
  if (tag=="ZZ" || tag=="") { // just add onto querystring
       querystring +=  query;
  }
  else {
    querystring += tag+":("+query+")";
  }
}


void addqueryelem_ex(text_t &querystring, const text_t &tag, 
             const text_t &terms, const text_t &stem, const text_t &fold,
             const text_t& combine, const text_t& word_combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
  }
  text_t outtext; outtext.reserve(512);
  text_t word; word.reserve(100);
  //unsigned short c;                                                            
  text_t::const_iterator here = terms.begin();
  text_t::const_iterator end = terms.end();
  bool inquote = false, firstword = true;

  text_t word2; word2.reserve(256);
    
  while (here !=end) {
    if (is_unicode_space(*here)) {
      if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
      else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
      else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
      else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
      else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
      if (inquote) {
    word2.push_back(*here);
      }
      word.append(word2); word2.clear();
            
      if (!inquote && !word.empty() ) {
                // found word boundary   
                
    if (stem == "1" || fold =="1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
                    
      if (fold == "1") word += "i";
      //else word += "c";
    }
    if (firstword) {
      firstword = false;
    } else {
      outtext += " " + word_combine + " ";
    }
    outtext += "[" + word + "]:"+tag;
    word.clear();
      }
      ++here;
    } else if (*here == '\"') {
      word2.push_back(*here);
      inquote = !inquote;
      ++here;
    } else {
      // not word boundary
      word2.push_back(*here);
      ++here;    
    }
  }
    
  // get last word
  if (!word2.empty()) {
    if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
    else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
    else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
    else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
    else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
    word.append(word2); word2.clear();
        
    if (stem == "1"|| fold == "1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
            
      if (fold == "1") word += "i";
      //else word += "c";
    }
    if (!outtext.empty()) outtext += " " + word_combine + " ";
    outtext += "[" + word + "]:"+tag;
  }
  querystring += "(" + outtext + ")";
}


void add_field_info(text_t &querystring, const text_t &tag, int type) {

  if (tag == "") return; // do nothing
  if (type == 1) { //mgpp
    querystring = "["+querystring+"]:"+tag;
  } else if (type == 2) { // lucene
    querystring = tag+":("+querystring+")";
  }
    
}

bool is_special_character(int indexer_type, unsigned short character) {
  // mgpp
  if (indexer_type == 1) {
    return (character == '#' || character == '/' || character == '*');
  }
  // lucene
  else if (indexer_type ==2) {
    return (character == '?' || character == '*' || character == '~' || 
        character == '^');
  }
  return false;
}

void format_field_info_lucene(text_t &querystring, cgiargsclass &args) {
  text_t tag = args["fqf"];
  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
  int type = 2; //lucene
  int argt = args.getintarg("t");// t=0 -and, t=1 - or
  int argb = args.getintarg("b"); // b=0 simple, b=1 advanced

  // lucene simple OR - the string stays as is, but may need field tag
  if (argb==0 && argt == 1) {
    // just tag the entire thing
    if (tag != "") {
      add_field_info(querystring, tag, type);
    }
    return;
  }
  bool in_phrase = false;
  
  text_t queryelem = "";
  text_t finalquery = "";
  
  // only add in + for simple AND search
  text_t combine = ((argb==0)? "+" : "");

  // for lucene, we need to change & to && and | to || if advanced search
  // we need to tag the entire string, if we have a field
  // if we are simple and search, then we put && in between words
  
  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();
  while (here != end) {
    if (is_unicode_letdig(*here) || is_special_character(type, *here)) {
      queryelem.push_back(*here);
    }

    // Detect phrase starts/finishes
    else if (*here == '"') {
      queryelem.push_back(*here);
      if (in_phrase == false) in_phrase = true;
      else {
    finalquery += combine + queryelem;
    queryelem.clear();
    in_phrase = false;
      }
    }

    // Found word boundary, in a phrase
    else if (in_phrase) {
      queryelem.push_back(*here);
    }
    // Word boundary, but not in a phrase
    else {
      if (*here == '&') {
    queryelem.push_back('&');
    queryelem.push_back('&');
      } else if (*here == '|') {
    queryelem.push_back('|');
    queryelem.push_back('|');
      } else {
    if (!queryelem.empty()) {
      finalquery += combine + queryelem;
      queryelem.clear();
    }
    finalquery.push_back(*here);
      }
    }

    ++here;
  }

  // Get last element
  if (!queryelem.empty()) {
    finalquery += combine + queryelem;
  }

  add_field_info(finalquery, tag, type);
  querystring = finalquery;
 cerr << "final query = "<<finalquery<<endl;
}

void format_field_info_mgpp(text_t &querystring, cgiargsclass &args) {
  
  text_t tag = args["fqf"];
  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)
 
  int argt = args.getintarg("t");// t=0 -and, t=1 - or
  int argb = args.getintarg("b"); // b=0 simple, b=1 advanced

  if (tag == "" && argb ==1) {
    return; // no field specifier, advanced mode, the query stays as written
  }

  int type = 1; // mgpp

  bool simple_and = (argb==0 && argt==0); 
  text_t finalquery = "";
  text_t fieldpart ="";
  text_t queryelem = "";
  bool in_phrase = false;
  bool in_field = false;

  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();
  while (here != end) {
    if (is_unicode_letdig(*here)  || *here == '&' || is_special_character(type, *here)) {
      queryelem.push_back(*here);
    }
    else if (*here == '|') {
      in_field = false;
    }
    else if (*here == '!' || *here == '(' || *here == ')') {
      if (!in_phrase) { // ignore these if in_phrase
    // output field, then output operator
    in_field = false;
    if (!queryelem.empty()) {
      if (!simple_and && !fieldpart.empty()) {
        add_field_info(fieldpart, tag, type);
        finalquery += fieldpart;
        finalquery.push_back(' ');
        fieldpart.clear();
      }
      fieldpart += queryelem;
    }
    if (!fieldpart.empty()) {
      add_field_info(fieldpart, tag, type);
      finalquery += fieldpart;
      finalquery.push_back(' ');
    }
    fieldpart.clear();
    queryelem.clear();
    finalquery.push_back(*here);
    finalquery.push_back(' ');
      }
    }
    else if (*here == '"') {
      queryelem.push_back(*here);
      if (in_phrase == false) in_phrase = true;
      else {
    in_phrase = false;
      }
    }

    // Found word boundary, in a phrase
    else if (in_phrase) {
      queryelem.push_back(*here);
    }
    // Found a word boundary
    else {
      if (!queryelem.empty()) {
    if (queryelem == "&") {
      in_field = true;
      queryelem.clear();
    }
    else if (starts_with(queryelem, "NEAR") || starts_with(queryelem, "WITHIN")) {
      
      if (argb==1) {
        // simple search, these not allowed
        in_field = true;
        fieldpart += queryelem;
        fieldpart.push_back(' ');
      }
      queryelem.clear();
      
    } 
    else {
      if (!simple_and && !in_field) {
        if (!fieldpart.empty()) {
          add_field_info(fieldpart, tag, type);
          finalquery += fieldpart;
          finalquery.push_back(' ');
          fieldpart.clear();
        } 
      }
      
      fieldpart += queryelem;
      fieldpart.push_back(' ');
      queryelem.clear();
    }
      }
    }
    ++here;
  }
  // at the end
  if (!queryelem.empty()) {
    if (!simple_and && !in_field && !fieldpart.empty()) {
      add_field_info(fieldpart, tag, type);
      finalquery += fieldpart;
      finalquery.push_back(' ');
      fieldpart.clear();
    }
    fieldpart += queryelem;
  }
  if (!fieldpart.empty()) {
    add_field_info(fieldpart, tag, type);
    finalquery += fieldpart;
    fieldpart.clear();
    finalquery.push_back(' ');
  }

  querystring  = finalquery;
  cerr << "final query = "<<finalquery<<endl;
}

void format_field_info(text_t &querystring, cgiargsclass &args) {
  int argct = args.getintarg("ct");
  if (argct == 1) {
    format_field_info_mgpp(querystring, args);
  } else if (argct == 2) {
    format_field_info_lucene(querystring, args);
  }
}