source: trunk/gsdl/src/recpt/querytools.cpp@ 11573

/**********************************************************************
 *
 * querytools.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

#include "querytools.h"
#include <ctype.h>
#include "unitool.h" // for is_unicode_letdig

// request.filterResultOptions and request.fields (if required) should
// be set from the calling code
void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring,
                  cgiargsclass &args) {

  request.filterName = "QueryFilter";

  OptionValue_t option;

  option.name = "Term";
  option.value = querystring;
  request.filterOptions.push_back (option);

  option.name = "QueryType";
  option.value = (args.getintarg("t")) ? "ranked" : "boolean";
  request.filterOptions.push_back (option);

  option.name = "MatchMode";
  option.value = (args.getintarg("t")) ? "some" : "all";
  request.filterOptions.push_back (option);

  option.name = "Casefold";
  option.value = (args.getintarg("k")) ? "true" : "false";
  request.filterOptions.push_back (option);

  option.name = "Stem";
  option.value = (args.getintarg("s")) ? "true" : "false";
  request.filterOptions.push_back (option);

  if (!args["h"].empty()) {
    option.name = "Index";
    option.value = args["h"];
    request.filterOptions.push_back (option);
  }

  if (!args["j"].empty()) {
    option.name = "Subcollection";
    option.value = args["j"];
    request.filterOptions.push_back (option);
  }

  if (!args["n"].empty()) {
    option.name = "Language";
    option.value = args["n"];
    request.filterOptions.push_back (option);
  }
  
  if (!args["g"].empty()) { // granularity for mgpp
    option.name = "Level";
    option.value = args["g"];
    request.filterOptions.push_back (option);
  }

  set_more_queryfilter_options (request, args);
}

void set_queryfilter_options (FilterRequest_t &request, const text_t &querystring1,
                  const text_t &querystring2, cgiargsclass &args) {

  set_queryfilter_options (request, querystring1, args);

  // fill in the second query if needed
  if (!args["cq2"].empty()) {
    OptionValue_t option;

    option.name = "CombineQuery";
    option.value = args["cq2"];
    request.filterOptions.push_back (option);
    
    option.name = "Term";
    option.value = querystring2;
    request.filterOptions.push_back (option);
    
    option.name = "QueryType";
    option.value = (args.getintarg("t")) ? "ranked" : "boolean";
    request.filterOptions.push_back (option);

    option.name = "Casefold";
    option.value = (args.getintarg("k")) ? "true" : "false";
    request.filterOptions.push_back (option);

    option.name = "Stem";
    option.value = (args.getintarg("s")) ? "true" : "false";
    request.filterOptions.push_back (option);

    if (!args["h2"].empty()) {
      option.name = "Index";
      option.value = args["h2"];
      request.filterOptions.push_back (option);
    }

    if (!args["j2"].empty()) {
      option.name = "Subcollection";
      option.value = args["j2"];
      request.filterOptions.push_back (option);
    }

    if (!args["n2"].empty()) {
      option.name = "Language";
      option.value = args["n2"];
      request.filterOptions.push_back (option);
    }
  }
  set_more_queryfilter_options (request, args);
}

void set_more_queryfilter_options (FilterRequest_t &request, cgiargsclass &args) {

  OptionValue_t option;
  int arg_m = args.getintarg("m");
  
  option.name = "Maxdocs";
  option.value = arg_m;
  request.filterOptions.push_back (option);

  //  option.name = "StartResults";
  //  option.value = args["r"];
  //  request.filterOptions.push_back (option);

  //  option.name = "EndResults";
  //  int endresults = args.getintarg("o") + (args.getintarg("r") - 1);
  //  if ((endresults > arg_m) && (arg_m != -1)) endresults = arg_m;
  //  option.value = endresults;
  //  request.filterOptions.push_back (option);
}

void format_querystring (text_t &querystring, int querymode, bool segment) {
  text_t formattedstring;

  if (querymode == 1 && !segment) return;
  
  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();

  // space is used to insert spaces between Chinese
  // characters. No space is needed before the first
  // Chinese character.
  bool space = false;

  // want to remove ()|!& from querystring so boolean queries are just
  // "all the words" queries (unless querymode is advanced)
  while (here != end) {
    if ((querymode == 0) && (*here == '(' || *here == ')' || *here == '|' ||
                 *here == '!' || *here == '&')) {
      formattedstring.push_back(' ');
    } else if (segment) {
      if ((*here >= 0x4e00 && *here <= 0x9fa5) ||
      (*here >= 0xf900 && *here <= 0xfa2d)) {
    // Chinese character
    if (!space) formattedstring.push_back (0x200b); // zero width space
    formattedstring.push_back (*here);
    formattedstring.push_back (0x200b);
    space = true;
      } else {
    
    // non-Chinese character
    formattedstring.push_back (*here);
    space = false;
    
      }
    
    } else {
      formattedstring.push_back (*here);
    }
    ++here;
  }
  querystring = formattedstring;
}


    
void add_dates(text_t &querystring, int startdate, int enddate, 
           int startbc, int endbc, int ct)
{
  if(startdate)
    {
      int querystringis = 0;
      text_t::const_iterator here = querystring.begin();
      text_t::const_iterator end = querystring.end();
      while(here!=end)
    {
      if(!(isspace((*here)))){
        here = end;
        querystringis = 1;
      }
      else
        ++here;
    }
      //converting BCE dates
      if(startbc && startdate > 0)
    {
      startdate *= -1;
    }
      if(endbc && enddate > 0)
    {
      enddate *= -1;
    }
       if(enddate != 0 && enddate<startdate)
    {
      cout<<"enddate too small"<<endl;
      return;
    }
       if(querystringis)
     querystring.appendcstr(" AND");
       if(!enddate)
     {
       if (ct==1) {
         mgpp_adddateelem(querystring,startdate);
       }
       else { // lucene
         lucene_adddateelem(querystring,startdate);
       }
     }
       else{
     int nextdate = startdate;
     querystring.appendcstr(" (");
     while(nextdate<=enddate)
       {
         if(nextdate!=0) {
           if (ct==1) {
         mgpp_adddateelem(querystring,nextdate);
           }
           else { // lucene
         lucene_adddateelem(querystring,nextdate);
           }
         }
         ++nextdate;
       }
     querystring.appendcstr(" )");
       }
    }

}

// search history tool 
// also used for form query macros
text_t escape_quotes(const text_t &querystring) {

  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();
 
  text_t escquery = "";
  while (here != end) {
    if (*here != '\'' && *here != '\"' && *here != '\n' && *here != '\r') escquery.push_back(*here);
    else if (*here == '\n' || *here == '\r') {
      escquery.push_back(' ');
    } else {
      escquery +="\\\\";
      escquery.push_back(*here);
    }

    ++here;
  }
  return escquery;

}

// some query form parsing functions for use with mgpp

void parse_reg_query_form(text_t &querystring, cgiargsclass &args)
{
  querystring.clear();

  const int ct = args.getintarg("ct");
  int argt = args.getintarg("t");// t=0 -and, t=1 - or

  text_t combine;
  if (ct==1) {
    if (argt == 0) combine = "&";
    else combine = "|";
  }
  else { // lucene
    if (argt == 0) combine = "AND";
    else combine = "OR";
  }
  
  text_t field = args["fqf"];
  if (field.empty()) return; // no query
  text_tarray fields;
  splitchar(field.begin(), field.end(), ',', fields); 
  
  text_t value = args["fqv"];
  if (value.empty()) return; // somethings wrong
  text_tarray values;
  splitchar(value.begin(), value.end(), ',', values);


  for (int i=0; i< values.size(); ++i) {
    if (!values[i].empty()) {
      if (ct == 1) {
    mgpp_addqueryelem(querystring, fields[i], values[i], combine);
      }
      else { // lucene
    lucene_addqueryelem(querystring, fields[i], values[i], combine);
      }
    }
  }
  
}


void parse_adv_query_form(text_t &querystring, cgiargsclass &args){

  querystring.clear();

  const int ct = args.getintarg("ct");
  text_t combine;
  if (ct==1) {
    combine = "&";
  }
  else { // lucene
    combine = "AND";
  }

  text_t field = args["fqf"];
  if (field.empty()) return; // no query
  text_tarray fields;
  splitchar(field.begin(), field.end(), ',', fields); 
  
  text_t value = args["fqv"];
  if (value.empty()) return; // somethings wrong
  text_tarray values;
  splitchar(value.begin(), value.end(), ',', values);

  text_t stem = args["fqs"];
  if (stem.empty()) return; // somethings wrong
  text_tarray stems;
  splitchar(stem.begin(), stem.end(), ',', stems);

  text_t fold = args["fqk"];
  if (fold.empty()) return; // somethings wrong
  text_tarray folds;
  splitchar(fold.begin(), fold.end(), ',', folds);

  text_t comb = args["fqc"];
  if (comb.empty()) return; //somethings wrong
  text_tarray combs;
  splitchar(comb.begin(), comb.end(), ',', combs);
  
  for(int i=0; i< values.size(); ++i) {
    if (!values[i].empty()) {
      if (i!=0) {
    if (ct==1) {
      if (combs[i-1]=="and") combine = "&";
      else if (combs[i-1]=="or")combine = "|";
      else if (combs[i-1]=="not")combine = "!";
    }
    else { // lucene
      if (combs[i-1]=="and") combine = "AND";
      else if (combs[i-1]=="or")combine = "OR";
      else if (combs[i-1]=="not")combine = "NOT";
    }
      }
      text_t term = addstemcase(values[i], stems[i], folds[i]);
      mgpp_addqueryelem(querystring, fields[i], term, combine);
    }
    
  }
}

text_t addstemcase(const text_t &terms, const text_t &stem, const text_t &fold) {
  
  text_t outtext;
  text_t word;
  //unsigned short c;                                                            
  text_t::const_iterator here = terms.begin();
  text_t::const_iterator end = terms.end();

  while (here !=end) {

    if (is_unicode_letdig(*here)) {
      // not word boundary
      word.push_back(*here);
      ++here;    
    }
    else {
      // found word boundary   
      if (!word.empty() ) {
    if (stem == "1" || fold =="1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
      
      if (fold == "1") word += "i";
      //else word += "c";
    }
    
    word += " ";
    outtext += word;
    word.clear();
      }
      if (*here == '\"') {
    outtext.push_back(*here);
      }
      ++here;
    }
  }
   
  // get last word
  if (!word.empty()) {
    if (stem == "1"|| fold == "1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
      
      if (fold == "1") word += "i";
      //else word += "c";
    }
    word += " ";
    outtext += word;
  }
  return outtext;
}


void mgpp_adddateelem(text_t& querystring, const int date)
{
  querystring.appendcstr(" [");
  if(date<0) {
      querystring.appendcstr("bc");
      querystring.appendint((date*-1));
  }
  else {
    querystring.appendint(date);
  }
  querystring.appendcstr("]:CV");
}

void lucene_adddateelem(text_t& querystring, const int date)
{
  querystring.appendcstr(" CV:(");
  if(date<0) {
      querystring.appendcstr("bc");
      querystring.appendint((date*-1));
  }
  else {
    querystring.appendint(date);
  }
  querystring.appendcstr(")");
}


void mgpp_addqueryelem(text_t &querystring, text_t &tag, 
          text_t &query, text_t &combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
 
  }
  if (tag=="ZZ" || tag=="") { // just add onto querystring
       querystring +=  query;
  }
  else {
    querystring += "["+query+"]:"+tag;
  }

}

void lucene_addqueryelem(text_t &querystring, text_t &tag, 
          text_t &query, text_t &combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
 
  }
  if (tag=="ZZ" || tag=="") { // just add onto querystring
       querystring +=  query;
  }
  else {
    querystring += tag+":("+query+")";
  }
}


void addqueryelem_ex(text_t &querystring, const text_t &tag, 
             const text_t &terms, const text_t &stem, const text_t &fold,
             const text_t& combine, const text_t& word_combine) {
  if (!querystring.empty()) { // have to put and/or
    querystring += " " + combine + " ";
  }
  text_t outtext; outtext.reserve(512);
  text_t word; word.reserve(100);
  //unsigned short c;                                                            
  text_t::const_iterator here = terms.begin();
  text_t::const_iterator end = terms.end();
  bool inquote = false, firstword = true;

  text_t word2; word2.reserve(256);
    
  while (here !=end) {
    if (is_unicode_space(*here)) {
      if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
      else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
      else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
      else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
      else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
      if (inquote) {
    word2.push_back(*here);
      }
      word.append(word2); word2.clear();
            
      if (!inquote && !word.empty() ) {
                // found word boundary   
                
    if (stem == "1" || fold =="1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
                    
      if (fold == "1") word += "i";
      //else word += "c";
    }
    if (firstword) {
      firstword = false;
    } else {
      outtext += " " + word_combine + " ";
    }
    outtext += "[" + word + "]:"+tag;
    word.clear();
      }
      ++here;
    } else if (*here == '\"') {
      word2.push_back(*here);
      inquote = !inquote;
      ++here;
    } else {
      // not word boundary
      word2.push_back(*here);
      ++here;    
    }
  }
    
  // get last word
  if (!word2.empty()) {
    if (word2 == "AND") { word2.clear(); word2.push_back(7527); word2.appendcarr("AND", 3); word2.push_back(7527); }
    else if (word2 == "OR") { word2.clear(); word2.push_back(7527); word2.appendcarr("OR", 2); word2.push_back(7527); }
    else if (word2 == "NOT") { word2.clear(); word2.push_back(7527); word2.appendcarr("NOT", 3); word2.push_back(7527); }
    else if (word2 == "NEAR") { word2.clear(); word2.push_back(7527); word2.appendcarr("NEAR", 4); word2.push_back(7527); }
    else if (word2 == "WITHIN") { word2.clear(); word2.push_back(7527); word2.appendcarr("WITHIN", 6); word2.push_back(7527); }
    word.append(word2); word2.clear();
        
    if (stem == "1"|| fold == "1") {
      word += "#";
      if (stem == "1") word += "s";
      //else word += "u";
            
      if (fold == "1") word += "i";
      //else word += "c";
    }
    if (!outtext.empty()) outtext += " " + word_combine + " ";
    outtext += "[" + word + "]:"+tag;
  }
  querystring += "(" + outtext + ")";
}


void add_field_info(text_t &querystring, const text_t &tag, int type) {

  if (tag == "") return; // do nothing
  if (type == 1) { //mgpp
    querystring = "["+querystring+"]:"+tag;
  } else if (type == 2) { // lucene
    querystring = tag+":("+querystring+")";
  }
    
}

bool is_special_character(int indexer_type, unsigned short character) {
  // mgpp
  if (indexer_type == 1) {
    return (character == '#' || character == '/' || character == '*');
  }
  // lucene
  else if (indexer_type ==2) {
    return (character == '?' || character == '*' || character == '~' || 
        character == '^');
  }
  return false;
}

void format_field_info(text_t &querystring, cgiargsclass &args) {
  
  text_t tag = args["fqf"];
  if (tag == "ZZ") tag = ""; // ZZ is a special tag meaning no tag (all fields)

  int argct = args.getintarg("ct");
  bool mgpp = (argct == 1);
  bool lucene = (argct == 2);

  if (mgpp && tag == "") {
    return; // no field specifier: do nothing
  }
  
  int argt = args.getintarg("t");// t=0 -and, t=1 - or
  int argb = args.getintarg("b"); // b=0 simple, b=1 advanced

  bool simple_AND_search = (argb==0 && argt==0); 
  bool simple_OR_search = (argb==0 && argt==1);

  if (mgpp && simple_AND_search) {
    // mgpp, simple AND search, tag the whole query string
    add_field_info(querystring, tag, argct);
    return;
  }
  // resulting mgpp case - we need to tag each individual term or phrase
  // TODO - allow AND. OR in query string and don't tag these words

  if (lucene && (simple_OR_search || argb == 1)) {
      // OR search or advanced search (here we assume that the user has added their term mods - don't need to add term mods
    if (tag != "") {
      // tag the whole string
      add_field_info(querystring, tag, argct);
    }
    return;
  }
  

  // if we have got here, we need to add in combiners (lucene) or
  // we need to tag each individual word (mgpp OR search - mgpp can't do OR inside a field)

  text_t combine = ((lucene)? "+" : "");
  
  text_t processed_querystring = "";
  text_t queryelement = "";
  
  bool in_phrase = false;
  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();
  while (here != end) {
    if (is_unicode_letdig(*here) || is_special_character(argct, *here)) {
      queryelement.push_back(*here);
    }

    // Detect phrase starts/finishes
    else if (*here == '"') {
      queryelement.push_back(*here);
      if (in_phrase == false) in_phrase = true;
      else {
    if (mgpp) {add_field_info(queryelement, tag, argct);}
    processed_querystring += combine + queryelement;
    queryelement.clear();
    in_phrase = false;
      }
    }

    // Found word boundary, in a phrase
    else if (in_phrase) {
      queryelement.push_back(*here);
    }
    // Word boundary, but not in a phrase
    else {
      if (!queryelement.empty()) {
    if (mgpp) {add_field_info(queryelement, tag, argct);}
    processed_querystring += combine + queryelement;
    queryelement.clear();
      }
      processed_querystring.push_back(*here);
    }

    ++here;
  }

  // Get last element
  if (!queryelement.empty()) {
    if (mgpp) {add_field_info(queryelement, tag, argct);}
    processed_querystring += combine + queryelement;
  }

  querystring = processed_querystring;
 
  if (lucene) {
    // tag the whole query string
    add_field_info(querystring, tag, argct);
  }
}