/**********************************************************************
 *
 * queryaction.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

#include "queryaction.h"
#include "querytools.h"
#include "formattools.h"
#include "cgiutils.h"
#include "OIDtools.h"
//#include "infodbclass.h"
#include "fileutil.h"
#include "text_t.h"
#include "historydb.h"
#include "htmlutils.h" // for html_safe in do_action
#include "gsdltools.h"
#include "phrases.h" // for get_phrases
#include <stdlib.h> // for strtol
#include <assert.h>

void colinfo_t::clear () {
  formatlistptr = NULL;
  browserptr = NULL;
}

void QueryResult_t::clear() {
  doc.clear();
  collection.clear();
}

queryaction::queryaction () {

  recpt = NULL;
  num_phrases = 0;

  // this action uses cgi variable "a"
  cgiarginfo arg_ainfo;
  arg_ainfo.shortname = "a";
  arg_ainfo.longname = "action";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "q";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  
   // "ct" - 0 = mg, 1 = mgpp, 2=lucene
  arg_ainfo.shortname = "ct";
  arg_ainfo.longname = "collection type";
  arg_ainfo.multiplechar = true; // can be empty or single char
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
 
  // "b" - 0 = simple, 1 = advanced
  arg_ainfo.shortname = "b";
  arg_ainfo.longname = "query mode";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "h"
  arg_ainfo.shortname = "h";
  arg_ainfo.longname = "main index";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "h2"
  arg_ainfo.shortname = "h2";
  arg_ainfo.longname = "main index for second query";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "j"
  arg_ainfo.shortname = "j";
  arg_ainfo.longname = "sub collection index";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "j2"
  arg_ainfo.shortname = "j2";
  arg_ainfo.longname = "sub collection index for second query";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "n"
  arg_ainfo.shortname = "n";
  arg_ainfo.longname = "language index";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "n2"
  arg_ainfo.shortname = "n2";
  arg_ainfo.longname = "language index for second query";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);


  // "q"
  arg_ainfo.shortname = "q";
  arg_ainfo.longname = "query string";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "q2"
  arg_ainfo.shortname = "q2";
  arg_ainfo.longname = "query string for second query";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "cq2" ""=don't combine, "and", "or", "not"
  arg_ainfo.shortname = "cq2";
  arg_ainfo.longname = "combine queries";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "t" - 1 = ranked 0 = boolean
  arg_ainfo.shortname = "t";
  arg_ainfo.longname = "search type";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "1";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "k"
  arg_ainfo.shortname = "k";
  arg_ainfo.longname = "casefolding";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "1";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ks"
  arg_ainfo.shortname = "ks";
  arg_ainfo.longname = "casefolding support";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  
  // "s"
  arg_ainfo.shortname = "s";
  arg_ainfo.longname = "stemming";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ss"
  arg_ainfo.shortname = "ss";
  arg_ainfo.longname = "stemming support";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "af"
  arg_ainfo.shortname = "af";
  arg_ainfo.longname = "accentfolding";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "afs"
  arg_ainfo.shortname = "afs";
  arg_ainfo.longname = "accentfolding support";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
 
  // "m"
  arg_ainfo.shortname = "m";
  arg_ainfo.longname = "maximum number of documents";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "50";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "o"
  arg_ainfo.shortname = "o";
  arg_ainfo.longname = "hits per page";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "20";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "r"
  arg_ainfo.shortname = "r";
  arg_ainfo.longname = "start results from";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "1";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ccs"
  arg_ainfo.shortname = "ccs";
  arg_ainfo.longname = "cross collection searching";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ccp"
  arg_ainfo.shortname = "ccp";
  arg_ainfo.longname = "cross collection page";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "cc"
  arg_ainfo.shortname = "cc";
  arg_ainfo.longname = "collections to search";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "hd" history display  - search history only displayed when
  // this var set to something other than 0
  // this number of records is displayed
  arg_ainfo.shortname = "hd";
  arg_ainfo.longname = "history display";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
	    
  // "hs"  save - set to 1 in query form, so only save when submit
  // query
  // 0 = no save 1 = save
  arg_ainfo.shortname = "hs";
  arg_ainfo.longname = "history save";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "g" - new arg for granularity, for mgpp collections
  arg_ainfo.shortname = "g";
  arg_ainfo.longname = "granularity";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText; 
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ds" - start date
  arg_ainfo.shortname = "ds";
  arg_ainfo.longname = "start date";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "de" - end date
  arg_ainfo.shortname = "de";
  arg_ainfo.longname = "end date";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "dsbc" - whether or not start date is prechristian
  arg_ainfo.shortname = "dsbc";
  arg_ainfo.longname = "start date bc";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "debc" - whether or not end date is prechristian
  arg_ainfo.shortname = "debc";
  arg_ainfo.longname = "end date bc";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "qt" - 0 = text, 1 = form
  arg_ainfo.shortname = "qt";
  arg_ainfo.longname = "query type";
  arg_ainfo.multiplechar = true; // can be empty or single char
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "qto" - 1 = text only, 2 = form only, 3 = text and form
  arg_ainfo.shortname = "qto";
  arg_ainfo.longname = "query type options";
  arg_ainfo.multiplechar = true; // can be empty or single char
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
    
  // "qb" - 0 = regular, 1 = large
  arg_ainfo.shortname = "qb";
  arg_ainfo.longname = "query box type";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "fqn" - number of fields in the query form
  arg_ainfo.shortname = "fqn";
  arg_ainfo.longname = "form query num fields";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "4";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "fqf" - the list of field names in the form query
  // - a comma separated list
  arg_ainfo.shortname = "fqf";
  arg_ainfo.longname = "form query fields";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  
  // "fqv" - the list of values in the form query
  // - a comma separated list
  arg_ainfo.shortname = "fqv";
  arg_ainfo.longname = "form query values";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  

  // "fqs" - the list of stemming options in the form query
  // - a comma separated list
  arg_ainfo.shortname = "fqs";
  arg_ainfo.longname = "form query stems";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  

  // "fqk" - the list of casefolding options in the form query
  // - a comma separated list
  arg_ainfo.shortname = "fqk";
  arg_ainfo.longname = "form query casefolds";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
  
  // "fqc" - the list of boolean operators in the form query
  // - a comma separated list
  arg_ainfo.shortname = "fqc";
  arg_ainfo.longname = "form query combines";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "fqa" - form query advanced - for "run query"
  arg_ainfo.shortname = "fqa";
  arg_ainfo.longname = "form query advanced query";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ifl" - I'm feeling lucky! (Go directly to a matching document)
  arg_ainfo.shortname = "ifl";
  arg_ainfo.longname = "i'm feeling lucky";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "ifln" - I'm feeling lucky number (Go directly to the nth matching document)
  arg_ainfo.shortname = "ifln";
  arg_ainfo.longname = "i'm feeling lucky number";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "1";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "srn" - the next search result
  arg_ainfo.shortname = "srn";
  arg_ainfo.longname = "the next search result number";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "srp" - the previous search result
  arg_ainfo.shortname = "srp";
  arg_ainfo.longname = "the previous search result number";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "sf" - Sort field. Set to field to be used for sorting search reult
  // set (only implemented for lucene collections at present).
  arg_ainfo.shortname = "sf";
  arg_ainfo.longname = "sort field";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // "fuzziness" controls how closely the search terms must match
  // 100 = exact match, 0 = very inexact match (only implemented for Lucene)
  arg_ainfo.shortname = "fuzziness";
  arg_ainfo.longname = "Lucene fuzziness value";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = g_EmptyText;
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
}

void queryaction::configure (const text_t &key, const text_tarray &cfgline) {
  action::configure (key, cfgline);
}

bool queryaction::init (ostream &logout) {
  return action::init (logout);
}

bool queryaction::check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
				 recptprotolistclass * /*protos*/, ostream &logout) {

  // check t argument
  int arg_t = args.getintarg("t");
  if (arg_t != 0 && arg_t != 1) {
    logout << "Warning: \"t\" argument out of range (" << arg_t << ")\n";
    cgiarginfo *tinfo = argsinfo.getarginfo ("t");
    if (tinfo != NULL) args["t"] = tinfo->argdefault;
  }

  // check k argument
  int arg_k = args.getintarg("k");
  if (arg_k != 0 && arg_k != 1) {
    logout << "Warning: \"k\" argument out of range (" << arg_k << ")\n";
    cgiarginfo *kinfo = argsinfo.getarginfo ("k");
    if (kinfo != NULL) args["k"] = kinfo->argdefault;
  }

  // check s argument
  int arg_s = args.getintarg("s");
  if (arg_s != 0 && arg_s != 1) {
    logout << "Warning: \"s\" argument out of range (" << arg_s << ")\n";
    cgiarginfo *sinfo = argsinfo.getarginfo ("s");
    if (sinfo != NULL) args["s"] = sinfo->argdefault;
  }

  // check m argument
  int arg_m = args.getintarg("m");
  if (arg_m < -1) {
    logout << "Warning: \"m\" argument less than -1 (" << arg_m << ")\n";
    cgiarginfo *minfo = argsinfo.getarginfo ("m");
    if (minfo != NULL) args["m"] = minfo->argdefault;
  }

  // check o argument
  int arg_o = args.getintarg("o");
  if (arg_o < -1) {
    logout << "Warning: \"o\" argument less than -1 (" << arg_o << ")\n";
    cgiarginfo *oinfo = argsinfo.getarginfo ("o");
    if (oinfo != NULL) args["o"] = oinfo->argdefault;
  }

  // check r argument
  int arg_r = args.getintarg("r");
  if (arg_r < 1) {
    logout << "Warning: \"r\" argument less than 1 (" << arg_r << ")\n";
    cgiarginfo *rinfo = argsinfo.getarginfo ("r");
    if (rinfo != NULL) args["r"] = rinfo->argdefault;
  }
  //check hd argument
  int arg_hd = args.getintarg("hd");
  if (arg_hd <0 ) {
    logout << "Warning: \"hd\" argument less than 0 (" << arg_hd << ")\n";
    cgiarginfo *hdinfo = argsinfo.getarginfo ("hd");
    if (hdinfo != NULL) args["hd"] = hdinfo->argdefault;
  }  
  
  //check hs argument
  int arg_hs = args.getintarg("hs");
  if (arg_hs !=0 && arg_hs !=1) {
    logout << "Warning: \"hs\" argument out of range (" << arg_hs << ")\n";
    cgiarginfo *hsinfo = argsinfo.getarginfo ("hs");
    if (hsinfo != NULL) args["hs"] = hsinfo->argdefault;
  }

  // check ct argument
  int arg_ct = args.getintarg("ct");
  if (arg_ct < 0 || arg_ct > 2) {
    logout << "Warning: \"ct\" argument out of range (" << arg_ct << ")\n";
    cgiarginfo *ctinfo = argsinfo.getarginfo ("ct");
    if (ctinfo != NULL) args["ct"] = ctinfo->argdefault;
  }

  // check qt argument
  int arg_qt = args.getintarg("qt");
  if (arg_qt !=0 && arg_qt !=1) {
    logout << "Warning: \"qt\" argument out of range (" << arg_qt << ")\n";
    cgiarginfo *qtinfo = argsinfo.getarginfo ("qt");
    if (qtinfo != NULL) args["qt"] = qtinfo->argdefault;
  }

  // check qb argument
  int arg_qb = args.getintarg("qb");
  if (arg_qb !=0 && arg_qb !=1) {
    logout << "Warning: \"qb\" argument out of range (" << arg_qb << ")\n";
    cgiarginfo *qbinfo = argsinfo.getarginfo ("qb");
    if (qbinfo != NULL) args["qb"] = qbinfo->argdefault;
  }

  // check fqa argument
  int arg_fqa = args.getintarg("fqa");
  if (arg_fqa !=0 && arg_fqa !=1) {
    logout << "Warning: \"fqa\" argument out of range (" << arg_fqa << ")\n";
    cgiarginfo *fqainfo = argsinfo.getarginfo ("fqa");
    if (fqainfo != NULL) args["fqa"] = fqainfo->argdefault;
  }

  // check fqn argument
  int arg_fqn = args.getintarg("fqn");
  if (arg_fqn < -1) {
    logout << "Warning: \"fqn\" argument less than -1 (" << arg_fqn << ")\n";
    cgiarginfo *fqninfo = argsinfo.getarginfo ("fqn");
    if (fqninfo != NULL) args["fqn"] = fqninfo->argdefault;
  }

  return true;
}

void queryaction::get_cgihead_info (cgiargsclass &args, recptprotolistclass * /*protos*/,
				    response_t &response, text_t &response_data, 
				    ostream &/*logout*/) {
  // If this is an "I'm feeling lucky" request, we don't know the target location until later
  if (!args["ifl"].empty()) {
    response = undecided_location;
    return;
  }

  response = content;
  response_data = "text/html";
}

void queryaction::define_internal_macros (displayclass &disp, cgiargsclass &args, 
					  recptprotolistclass * protos, 
					  ostream &logout) {

  // define_internal_macros sets the following macros:

  // The following macros are set later (in define_query_macros) as they can't be set until 
  // the query has been done.
  // _quotedquery_   the part of the query string that was quoted for post-processing
  // _freqmsg_      the term frequency string

  // _resultline_   the "x documents matched the query" string

  // _prevfirst_    these are used when setting up the links to previous/next
  // _prevlast_     pages of results (_thisfirst_ and _thislast_ are used to set
  // _nextfirst_    the 'results x-x for query: xxxx' string in the title bar)
  // _nextlast_
  // _thisfirst_
  // _thislast_

  
  define_form_macros(disp, args, protos, logout);
  
  define_query_interface(disp, args, protos, logout);


}

void queryaction::define_query_interface(displayclass &disp, 
					 cgiargsclass &args, 
					 recptprotolistclass * protos, 
					 ostream &logout){
  text_t collection = args["c"];

  //check that the protocol is alive
  recptproto* colproto = protos->getrecptproto (collection, logout);
  if(colproto == NULL) {
    logout << "ERROR: Null collection protocol trying to query"
	   << collection.getcstr() << "\n";
    return;
  }
  
  //check the collection is responding/in place
  ColInfoResponse_t *colinfo = recpt->get_collectinfo_ptr(colproto, collection,
							  logout);
  if(colinfo == NULL){
    logout << "ERROR: Null returned for get_collectinfo_ptr on "
	   << collection.getcstr() << "in queryaction::define_query_interface\n";
    return;
  }
   
  text_tmap::iterator check = colinfo->format.find("QueryInterface");
  if(check != colinfo->format.end()){
    if((*check).second=="DateSearch"){
      text_t current = "_datesearch_";
      disp.setmacro("optdatesearch","query",current);
    }
  }
}


// sets the selection box macros _hselection_, _jselection_, _nselection_ _gselection_, fqfselection_ 
void queryaction::set_option_macro (const text_t &macroname, 
				    text_t current_value, 
				    bool display_single, 
				    bool add_js_update,
				    const FilterOption_t &option, 
				    displayclass &disp) {
  
  if (option.validValues.empty()) return;
  if (option.validValues.size() == 1) {
    if (display_single) {
      disp.setmacro (macroname + "selection", displayclass::defaultpackage, "_" + option.defaultValue + "_");
    }
    return;    
  } 
  if (option.validValues.size() < 2) return;
  
  text_t macrovalue = "<select name=\"" + macroname + "\"";
  if (add_js_update) {
    macrovalue += " onChange=\"update"+macroname+"();\"";
  }
  macrovalue += ">\n";
  
  if (current_value.empty()) current_value = option.defaultValue;
  
  text_tarray::const_iterator thisvalue = option.validValues.begin();
  text_tarray::const_iterator endvalue = option.validValues.end();

  while (thisvalue != endvalue) {
    macrovalue += "<option value=\"" + *thisvalue + "\"";
    if (*thisvalue == current_value)
      macrovalue += " selected";
    macrovalue += ">_" + *thisvalue + "_\n";
    ++thisvalue;
  }
  macrovalue += "</select>\n";
  disp.setmacro (macroname + "selection", displayclass::defaultpackage, macrovalue);
}


void queryaction::define_external_macros (displayclass &disp, cgiargsclass &args, 
					  recptprotolistclass *protos, ostream &logout) {

  // define_external_macros sets the following macros:

  // some or all of these may not be required to be set
  // _hselection_, _h2selection_  the selection box for the main part of the index
  // _jselection_, _j2selection_  the selection box for the subcollection part of the index
  // _nselection_, _n2selection_  the selection box for the language part of the index
  // _cq2selection                the selection box for combining two queries
  
  // _gselection_, the selection box forlevels (mgpp)
  // _fqfselection_, the selection box for index/fields (mgpp)
  // can't do anything if collectproto is null (i.e. no collection was specified)
  recptproto *collectproto = protos->getrecptproto (args["c"], logout);
  if (collectproto == NULL) return;

  ColInfoResponse_t *colinfo = recpt->get_collectinfo_ptr(collectproto, 
							  args["c"],
							  logout);
  set_query_type_args(colinfo, args);
  set_stem_index_args(colinfo, args);

  comerror_t err;
  InfoFilterOptionsResponse_t response;
  InfoFilterOptionsRequest_t request;
  request.filterName = "QueryFilter";
  
  collectproto->get_filteroptions (args["c"], request, response, err, logout);
  if (err == noError) {
    
    FilterOption_tmap::const_iterator it;
    FilterOption_tmap::const_iterator end = response.filterOptions.end();
    
    // _hselection_ and _h2selection_ (Index)
    it = response.filterOptions.find ("Index");
    if (it != end) set_option_macro ("h", args["h"], true, false, (*it).second, disp);
    if (it != end) set_option_macro ("h2", args["h2"], true,false, (*it).second, disp);
    
    // _jselection_ and _j2selection_ (Subcollection)
    it = response.filterOptions.find ("Subcollection");
    if (it != end) set_option_macro ("j", args["j"], true,false, (*it).second, disp);
    if (it != end) set_option_macro ("j2", args["j2"], true,false, (*it).second, disp);
    
    // _nselection_ and _n2selection_ (Language)
    it = response.filterOptions.find ("Language");
    if (it != end) set_option_macro ("n", args["n"], true,false, (*it).second, disp);
    if (it != end) set_option_macro ("n2", args["n2"], true,false, (*it).second, disp);

    // _cq2selection_ (CombineQuery)
    it = response.filterOptions.find ("CombineQuery");
    if (it != end) set_option_macro ("cq2", args["cq2"], true,false, (*it).second, disp);

    if ((args["ct"] == "1") || (args["ct"] == "2"))  { // mgpp/lucene collections
      // _gselection_ (Level) 
      it = response.filterOptions.find("Level");
      if (it!=end) {
	set_option_macro("g", args["g"], false, false, (*it).second, disp);
	if (args["qt"]=="1") { // form search
	  set_gformselection_macro(args["g"], (*it).second, disp);
	}
      }
      // _fqfselection_ field list
      it = response.filterOptions.find("IndexField");
      if (it!=end) {
	bool form_search = false;
	if (args["qto"]=="2" || args["qt"]=="1") { 
	  form_search = true;
	}
	set_option_macro ("fqf", args["fqf"], true, form_search, (*it).second, disp);
	if (args["ct"] == "2") {// lucene
	  // set the sort field macro
	  set_sfselection_macro(args["sf"], (*it).second, disp);
	}
      }
    }
  }
} // define external macros
  
void queryaction::set_sfselection_macro(text_t current_value, 
				       const FilterOption_t &option, 
				       displayclass &disp) {

  // we need two or more options to continue
  if (option.validValues.size() <= 2) {
    return;
  }

  text_t macrovalue = "<select name=\"sf\">\n";
  
  if (current_value.empty()) current_value = "";
  
  // we give a rank option first
  macrovalue += "<option value=\"\"";
  if (current_value == "") {
    macrovalue += " selected";
  }
  macrovalue += ">_query:textsortbyrank_\n";

  text_tarray::const_iterator thisvalue = option.validValues.begin();
  text_tarray::const_iterator endvalue = option.validValues.end();
  int valid_count = 0;
  while (thisvalue != endvalue) {
    if (*thisvalue != "ZZ" && *thisvalue != "TX") {
      ++valid_count;
      macrovalue += "<option value=\"by" + *thisvalue + "\"";
      if (current_value == "by"+*thisvalue)
	macrovalue += " selected";
      macrovalue += ">_" + *thisvalue + "_\n";
    }
    ++thisvalue;
  }
  macrovalue += "</select>";
  if (valid_count > 0) {
    disp.setmacro ("sfselection", displayclass::defaultpackage, macrovalue);
  }
  
}
 
// sets the selection box macro _gformselection_.
// the default for _gformselection_ is _gselection_
void queryaction::set_gformselection_macro (text_t current_value,
					const FilterOption_t &option,
					displayclass &disp) {
  
  if (option.validValues.size() <= 1) {
    return;
  }
  // we need to check to see if there is paragraph present
  text_tarray::const_iterator thisvalue = option.validValues.begin();
  text_tarray::const_iterator endvalue = option.validValues.end();

  bool has_paras = false;
  while (thisvalue != endvalue) {
    if (*thisvalue == "Para") {
      has_paras = true;
      break;
    }
    ++thisvalue;
  }
  if (!has_paras) return; // there is no difference between the form selection and the normal one

  if (option.validValues.size() == 2) {
    // we will only have one value, but we will still put it in as a text string
    int opt = 0;
    if (option.validValues[0] == "Para") {
      opt = 1;
    }
    disp.setmacro ("gformselection", displayclass::defaultpackage, "_"+option.validValues[opt]+"_");
    return;
  }

  // there will be a select box
  text_t macrovalue = "<select name=\"g\">\n";

  if (current_value.empty()) current_value = option.defaultValue;
  
  thisvalue = option.validValues.begin();
  
  while (thisvalue != endvalue) {
    if (*thisvalue != "Para") {
      macrovalue += "<option value=\"" + *thisvalue + "\"";
      if (*thisvalue == current_value)
	macrovalue += " selected";
      macrovalue += ">_" + *thisvalue + "_\n";
    }
    ++thisvalue;
  }
  macrovalue += "</select>\n";
  disp.setmacro ("gformselection", displayclass::defaultpackage, macrovalue);
}
void queryaction::define_form_macros (displayclass &disp, cgiargsclass &args, 
			   recptprotolistclass *protos, ostream &logout) {

  // defines the following macros
  // _regformlist_
  // _advformlist_

  if (args["ct"]=="0" || args["qto"]=="1" || (args["qto"]=="3" && args["qt"] == "0") ) // mg, or mgpp/lucene with plain only, or mgpp with both, but set to plain
    return; // dont need these macros

  text_t form = "";
  int argfqn = args.getintarg("fqn");
  
  if (args["b"] == "1") { // advanced form
    form += "_firstadvformelement_\n";
    for (int i=1; i<argfqn; ++i) {      
	form += "_advformelement_\n";
    }
    disp.setmacro("advformlist", "query", form);
  }
  else { // simple form
    for (int i=0; i<argfqn; ++i) {
      form += "_regformelement_\n"; 
    }
    disp.setmacro("regformlist", "query", form);
  }
  
}

void queryaction::define_history_macros (displayclass &disp, cgiargsclass &args, 
					 recptprotolistclass *protos, ostream &logout) {

  // defines the following macros
  // _searchhistorylist_

  text_t historylist;
  int arghd = args.getintarg("hd");
  if (arghd == 0) {
    historylist="";
  }
  else {
    historylist = "<!-- Search History List -->\n";
   
    text_t userid = args["z"];
    text_tarray entries;
    if (get_history_info (userid, entries, gdbmhome, logout)) {
      int count = 1;
      text_tarray::iterator here = entries.begin();
      text_tarray::iterator end = entries.end();
      int numrecords=(int)entries.size();
      if (numrecords>arghd) { // only display some of them 
	numrecords = arghd;
      }
      historylist += "<form action=\"_gwcgi_\" name=\"HistoryForm\"><table width=\"537\">\n";

      for (int i=0; i<numrecords;++i) {
	text_t query;
	text_t numdocs;
	text_t cgiargs;
	text_t userinfo;
	text_t escquery;
	split_saved_query(entries[i],numdocs,cgiargs);
	parse_saved_args(cgiargs, "q", query); // get query string out
	decode_cgi_arg(query); // un cgisafe it
	escquery = escape_quotes(query); // escape the quotes and newlines
	text_t histvalue = "histvalue";
	histvalue += i;
	disp.setmacro(histvalue, "query", escquery);
	format_user_info(cgiargs, userinfo, args, protos, logout);
	
	historylist += "<tr><td align=\"right\">_imagehistbutton_(";
	historylist += i;
	historylist += ")</td>\n";
	historylist += "<td><table border=\"1\" cellspacing=\"0\" ";
	historylist += "cellpadding=\"0\"><tr><td width=\"365\" align=\"left\">"
	  + query 
	  + "</td></tr></table></td><td width=\"110\" align=\"center\"><small>"
	  + numdocs;
	if (numdocs == 1) historylist += " _texthresult_";
	else historylist += " _texthresults_";
	if (!userinfo.empty()) {
	  historylist += "<br>( "+userinfo+" )";
	}
	historylist += "</small></td>\n";
      }
      historylist+="</table></form>\n\n";
     
    } // if get history info 
    else {
      historylist += "_textnohistory_";
    }
    historylist += "<! ---- end of history list ----->\n";
  } // else display list
  disp.setmacro("searchhistorylist", "query", historylist);
  
} // define history macros

void queryaction::output_ccp (cgiargsclass &args, recptprotolistclass *protos,
			      displayclass &disp, outconvertclass &outconvert,
			      ostream &textout, ostream &logout) {

  ColInfoResponse_t *cinfo = NULL;
  comerror_t err;
  InfoFilterOptionsResponse_t fresponse;
  InfoFilterOptionsRequest_t frequest;
  frequest.filterName = "QueryFilter";

  text_t &index = args["h"];
  text_t &subcollection = args["j"];
  text_t &language = args["n"];

  text_tset collections;
  text_t arg_cc = args["cc"];
  decode_cgi_arg (arg_cc);
  splitchar (arg_cc.begin(), arg_cc.end(), ',', collections);

  textout << outconvert << disp << "_query:header_\n"
	  << "<center>_navigationbar_</center><br>\n"
	  << "<form name=\"QueryForm\" method=\"get\" action=\"_gwcgi_\">\n"
	  << "<input type=\"hidden\" name=\"a\" value=\"q\">\n"
          << "<input type=\"hidden\" name=\"site\" value=\"_cgiargsite_\"\n"
	  << "<input type=\"hidden\" name=\"e\" value=\"_compressedoptions_\">\n"
	  << "<input type=\"hidden\" name=\"ccp\" value=\"1\">\n"
	  << "<center><table width=\"_pagewidth_\"><tr valign=\"top\">\n"
	  << "<td>Select collections to search for \"" << args["q"] 
	  << "\" <i>(index=" << index << " subcollection=" << subcollection
	  << " language=" << language << ")</i></td>\n"
	  << "<td><input type=\"submit\" value=\"_query:textbeginsearch_\"></td>\n"
	  << "</tr></table></center>\n"
	  << "<center><table width=\"_pagewidth_\">\n"
	  << "<tr><td>\n";
  
  recptprotolistclass::iterator rprotolist_here = protos->begin();
  recptprotolistclass::iterator rprotolist_end = protos->end();
  while (rprotolist_here != rprotolist_end) {
    if ((*rprotolist_here).p != NULL) {
      
      text_tarray collist;
      (*rprotolist_here).p->get_collection_list (collist, err, logout);
      if (err == noError) {
	text_tarray::iterator collist_here = collist.begin();
	text_tarray::iterator collist_end = collist.end();
	while (collist_here != collist_end) {
	  
	  cinfo = recpt->get_collectinfo_ptr ((*rprotolist_here).p, *collist_here, logout);
	  //	  if (err == noError && cinfo.isPublic && (cinfo.buildDate > 0)) {
	  if (cinfo != NULL && (cinfo->buildDate > 0)) {
	    
	    (*rprotolist_here).p->get_filteroptions (*collist_here, frequest, fresponse, err, logout);
	    if (err == noError) {
	      
	      FilterOption_tmap::const_iterator it;
	      FilterOption_tmap::const_iterator end = fresponse.filterOptions.end();
	      if (!index.empty()) {
		it = fresponse.filterOptions.find ("Index");
		if (it == end) {++collist_here; continue;}
		text_tarray::const_iterator there = (*it).second.validValues.begin();
		text_tarray::const_iterator tend = (*it).second.validValues.end();
		while (there != tend) {
		  if (*there == index) break;
		  ++there;
		}
		if (there == tend) {++collist_here; continue;}
	      }
	      if (!subcollection.empty()) {
		it = fresponse.filterOptions.find ("Subcollection");
		if (it == end) {++collist_here; continue;}
		text_tarray::const_iterator there = (*it).second.validValues.begin();
		text_tarray::const_iterator tend = (*it).second.validValues.end();
		while (there != tend) {
		  if (*there == subcollection) break;
		  ++there;
		}
		if (there == tend) {++collist_here; continue;}
	      }
	      if (!language.empty()) {
		it = fresponse.filterOptions.find ("Language");
		if (it == end) {++collist_here; continue;}
		text_tarray::const_iterator there = (*it).second.validValues.begin();
		text_tarray::const_iterator tend = (*it).second.validValues.end();
		while (there != tend) {
		  if (*there == language) break;
		  ++there;
		}
		if (there == tend) {++collist_here; continue;}
	      }
	    
	      // we've got a matching collection
	      textout << outconvert << "<input type=\"checkbox\"";

	      text_tset::const_iterator t = collections.find (*collist_here);
	      if (t != collections.end()) textout << outconvert << " checked";
	      
	      text_t collectionname = cinfo->get_collectionmeta("collectionname", args["l"]);
	      if (collectionname.empty()) {
		collectionname = *collist_here;
	      }
	      textout << outconvert << disp 
		      << " name=\"cc\" value=\"" << *collist_here << "\">"
		      << collectionname << "<br>\n";

	      
	    }
	  }
	  ++collist_here;
	}
      }
    }
    ++rprotolist_here;
  }
  textout << outconvert << disp 
	  << "</td></tr></table></center>\n"
	  << "</form>\n"
	  << "_query:footer_\n";
  
}

bool queryaction::do_action (cgiargsclass &args, recptprotolistclass *protos, 
			     browsermapclass *browsers, displayclass &disp, 
			     outconvertclass &outconvert, ostream &textout, 
			     ostream &logout) {
  
  if (recpt == NULL) {
    logout << "ERROR (queryaction::do_action): This action does not contain information\n"
	   << "      about any receptionists. The method set_receptionist was probably\n"
	   << "      not called from the module which instantiated this action.\n";
    return true;
  }



  if (args["ccs"] == "1") {
    if (!args["cc"].empty()) {
      // query the selected collections
      text_t::const_iterator b = args["cc"].begin();
      text_t::const_iterator e = args["cc"].end();
      if (findchar (b, e, ',') != e) {
	if (!search_multiple_collections (args, protos, browsers, disp, outconvert,
					  textout, logout)) return false;
	return true;
      } else {
	if (!search_single_collection (args, args["cc"], protos, browsers, disp, 
				       outconvert, textout, logout)) return false;
  	return true;
      }
    }
  }

  // simply query the current collection
  if (!search_single_collection (args, args["c"], protos, browsers, disp, 
				 outconvert, textout, logout)) return false;
  return true;
}

bool queryaction::search_multiple_collections (cgiargsclass &args, recptprotolistclass *protos,
					       browsermapclass *browsers, displayclass &disp,
					       outconvertclass &outconvert, ostream &textout,
					       ostream &logout) {

  text_tarray collections;
  
  text_t arg_cc = args["cc"];
  decode_cgi_arg (arg_cc);
  splitchar (arg_cc.begin(), arg_cc.end(), ',', collections);

  if (collections.empty()) {
    logout << "queryaction::search_multiple_collections: No collections "
	   << "set for doing multiple query - will search current collection\n";
    textout << outconvert << disp << "_query:textwarningnocollections_\n";
    return search_single_collection (args, args["c"], protos, browsers, disp, 
				     outconvert, textout, logout);
  }

  // queryaction uses "VList" browser to display results,
  // a queries clasification is "Search"
  text_t browsertype = "VList";
  text_t classification = "Search";

  QueryResult_tset results;
  map<text_t, colinfo_t, lttext_t> colinfomap;

  ColInfoResponse_t *cinfo = NULL;
  recptproto *collectproto = NULL;
  comerror_t err;
  FilterRequest_t request;
  FilterResponse_t response;
  request.filterResultOptions = FROID | FRmetadata | FRtermFreq | FRranking;
  text_t freqmsg = "_textfreqmsg1_";
  int numdocs = 0;
  isapprox isApprox = Exact;

  // what to do about segmentation for multiple colls??
  bool segment = false;
  text_t formattedstring = "";
  get_formatted_query_string(formattedstring, segment, args, disp, logout);

  if (formattedstring.empty()) {
    // dont bother doing a query if no query string
    define_history_macros (disp, args, protos, logout);
    textout << outconvert << disp << "_query:header_\n"
	  << "_query:content_";
    textout << outconvert << disp << "_query:footer_";

    return true;
  }
  bool syntax_error = false;

  set_queryfilter_options (request, formattedstring, args);

  // need to retrieve maxdocs matches for each collection
  // (will eventually want to tidy this up, do so caching etc.)
  OptionValue_t option;
  option.name = "StartResults";
  option.value = "1";
  request.filterOptions.push_back (option);
  
  option.name = "EndResults";
  option.value = args["m"];
  request.filterOptions.push_back (option);

  text_tarray::iterator col_here = collections.begin();
  text_tarray::iterator col_end = collections.end();

  map<text_t, int, lttext_t> termfreqs;
  
  // just check the main col for formatting info - use individual format statements, or the main one?
  
  browserclass *bptr = browsers->getbrowser (browsertype);
  
  text_t main_col = args["c"];
  cinfo = recpt->get_collectinfo_ptr (collectproto, main_col, logout);
  if (cinfo == NULL) {
    logout << "ERROR (query_action::search_multiple_collections): get_collectinfo_ptr returned NULL for '"<<main_col<<"'\n";
    return false;
  }
  
  bool use_main_col_format = false;
  if (cinfo->ccsOptions & CCSUniformSearchResultsFormatting) {
    use_main_col_format = true;
  }
 
  request.fields.erase (request.fields.begin(), request.fields.end());
  request.getParents = false;
  bptr->load_metadata_defaults (request.fields);
  
  text_t formatstring;
  format_t *formatlistptr = new format_t();
  if (use_main_col_format) {
    // just get one format for main coll and use it for each subcol
    if (!get_formatstring (classification, browsertype, 
			   cinfo->format, formatstring)) {
      formatstring = bptr->get_default_formatstring();
    }
    
    parse_formatstring (formatstring, formatlistptr, request.fields, request.getParents);
  }
  
  while (col_here != col_end) {

    collectproto = protos->getrecptproto (*col_here, logout);
    if (collectproto == NULL) {
      logout << outconvert << "queryaction::search_multiple_collections: " << *col_here 
	     << " collection has a NULL collectproto, ignoring\n";
      ++col_here;
      continue;
    }
    cinfo = recpt->get_collectinfo_ptr (collectproto, *col_here, logout);
    if (cinfo == NULL) {
      logout << "ERROR (query_action::search_multiple_collections): get_collectinfo_ptr returned NULL\n";
      ++col_here;
      continue;
    }

    if (!use_main_col_format) {
      request.fields.erase (request.fields.begin(), request.fields.end());
      request.getParents = false;
      bptr->load_metadata_defaults (request.fields);
      
    //browserclass *bptr = browsers->getbrowser (browsertype);

    // get the formatstring if there is one 
      if (!get_formatstring (classification, browsertype, 
			     cinfo->format, formatstring)) {
	formatstring = bptr->get_default_formatstring();
      }

      formatlistptr = new format_t();
      parse_formatstring (formatstring, formatlistptr, request.fields, request.getParents);
    }
    
    colinfo_t thiscolinfo;
    thiscolinfo.formatlistptr = formatlistptr;
    thiscolinfo.browserptr = bptr;
    colinfomap[*col_here] = thiscolinfo;
    
    // do the query
    collectproto->filter (*col_here, request, response, err, logout);
    if (err != noError && err != syntaxError) {
      outconvertclass text_t2ascii;
      logout << text_t2ascii
	     << "queryaction::search_multiple_collections: call to QueryFilter failed "
	     << "for " << *col_here << " collection (" << get_comerror_string (err) << ")\n";
      return false;
    }

    if (err == syntaxError) {
      syntax_error = true;
      freqmsg = "_textinvalidquery_";
      // assume the syntax will be invalid for all colls
      break;
    }
    if (response.error_message == "TOO_MANY_CLAUSES") {
      freqmsg = "_textlucenetoomanyclauses_";
      break;
    }
    if (isApprox == Exact)
      isApprox = response.isApprox;
    else if (isApprox == MoreThan)
      if (response.isApprox == Approximate)
	isApprox = response.isApprox;

    TermInfo_tarray::const_iterator this_term = response.termInfo.begin();
    TermInfo_tarray::const_iterator end_term = response.termInfo.end();
    while (this_term != end_term) {
      termfreqs[(*this_term).term] += (*this_term).freq;
      if ((col_here+1) == col_end) {
	freqmsg += (*this_term).term + ": " + termfreqs[(*this_term).term];
	if ((this_term+1) != end_term) freqmsg += ", ";
      }
      ++this_term;
    }

    if (response.numDocs > 0) {
      numdocs += response.numDocs;
    
      QueryResult_t thisresult;
      thisresult.collection = *col_here;
      ResultDocInfo_tarray::iterator doc_here = response.docInfo.begin();
      ResultDocInfo_tarray::iterator doc_end = response.docInfo.end();
      while (doc_here != doc_end) {
	thisresult.doc = *doc_here;
	results.insert (thisresult);
	++doc_here;
      }
    }
    ++col_here;
  } // for each coll
  
  text_t numdocs_t = numdocs;  
  args["nmd"] = numdocs_t;

  disp.setmacro ("freqmsg", "query", freqmsg);

  define_query_macros( args, disp, numdocs, isApprox);
  // save the query if appropriate
  save_search_history(args, numdocs, isApprox);
  define_history_macros (disp, args, protos, logout);

  textout << outconvert << disp << "_query:header_\n"
	  << "_query:content_";

  if (!syntax_error) {
    
    // now go through each result and output it
    QueryResult_tset::iterator res_here = results.begin();
    QueryResult_tset::iterator res_end = results.end();
    text_tset metadata; // empty !!
    bool getParents = false; // don't care !!
    bool use_table;
    ResultDocInfo_t thisdoc;
    format_t *formatlistptr = NULL;
    browserclass *browserptr = NULL;
    
    int count = 1;
    int firstdoc = args.getintarg("r");
    int hitsperpage = args.getintarg("o");
    int thislast = firstdoc + (hitsperpage - 1);
    
    // output results
    while (res_here != res_end) {
      if (count < firstdoc) {++count; ++res_here; continue;}
      if (count > thislast) break;
      
      formatlistptr = colinfomap[(*res_here).collection].formatlistptr;
      browserptr = colinfomap[(*res_here).collection].browserptr;
      thisdoc = (*res_here).doc;
      use_table = is_table_content (formatlistptr);
      
      collectproto = protos->getrecptproto ((*res_here).collection, logout);
      if (collectproto == NULL) {
	logout << outconvert << "queryaction::search_multiple_collections: " << (*res_here).collection
	       << " collection has a NULL collectproto, ignoring results\n";
	++res_here;
	continue;
      }
      
      browserptr->output_section_group (thisdoc, args, (*res_here).collection, 0, 
					formatlistptr, use_table, metadata, getParents, 
					collectproto, disp, outconvert, textout, logout);
      //    textout << outconvert << "(ranking: " << (*res_here).doc.ranking << ")\n";
      ++res_here;
      ++count;
    }
  }
  textout << outconvert << disp << "_query:footer_";
  
  // clean up the format_t pointers
  map<text_t, colinfo_t, lttext_t>::iterator here =  colinfomap.begin();
  map<text_t, colinfo_t, lttext_t>::iterator end =  colinfomap.end();
  while (here != end) {
    delete ((*here).second.formatlistptr);
    ++here;
  }
  return true;
}

bool queryaction::search_single_collection (cgiargsclass &args, const text_t &collection,
					    recptprotolistclass *protos, browsermapclass *browsers, 
					    displayclass &disp, outconvertclass &outconvert, 
					    ostream &textout, ostream &logout) {

  recptproto *collectproto = protos->getrecptproto (collection, logout);
  if (collectproto == NULL) {
    logout << outconvert << "queryaction::search_single_collection: " << collection
	   << " collection has a NULL collectproto\n";

    // Display the "this collection is not installed on this system" page
    disp.setmacro("cvariable", displayclass::defaultpackage, collection);
    disp.setmacro("content", "query", "<p>_textbadcollection_<p>");

    textout << outconvert << disp << "_query:header_\n"
	    << "_query:content_\n" << "_query:footer_\n";
    return true;
  }

  // queryaction uses "VList" browser to display results,
  // a queries clasification is "Search"
  text_t browsertype = "VList";
  text_t classification = "Search";

  comerror_t err;
  ColInfoResponse_t *cinfo = recpt->get_collectinfo_ptr (collectproto, collection, logout);

  if (cinfo == NULL) {
    logout << "ERROR (query_action::search_single_collection): get_collectinfo_ptr returned NULL\n";
    return false;
  }
    
  bool segment = cinfo->isSegmented;
  browserclass *bptr = browsers->getbrowser (browsertype);

  // get the formatstring if there is one 
  text_t formatstring;
  if (!get_formatstring (classification, browsertype, 
			 cinfo->format, formatstring)) {
    formatstring = bptr->get_default_formatstring();
  }
  FilterRequest_t request;
  FilterResponse_t response;

  text_t maxDoc_old = args["o"];
  text_t starts_old = args["r"];  

    //if the "ifl" argument is set ignore the "r" and "o" arg
  if(!args["ifl"].empty()){
    args["r"] = 1;
    args["o"] = args["m"];
  }
 

  bptr->set_filter_options (request, args);
  bptr->load_metadata_defaults (request.fields);

  format_t *formatlistptr = new format_t();
  parse_formatstring (formatstring, formatlistptr, request.fields, request.getParents);
    
  // do the query
  request.filterResultOptions = FROID | FRmetadata | FRtermFreq;
  text_t formattedstring = "";
  get_formatted_query_string(formattedstring, segment, args, disp, logout);


  if (!formattedstring.empty()) { // do the query
    // note! formattedstring is in unicode! mg and mgpp must convert!
    set_queryfilter_options (request, formattedstring, args);

    collectproto->filter (collection, request, response, err, logout);

    if (err != noError) {
      outconvertclass text_t2ascii;
      logout << text_t2ascii
	     << "queryaction::search_single_collections: call to QueryFilter failed "
	     << "for " << collection << " collection (" << get_comerror_string (err) << ")\n";
     
    }

    // Perform the "I'm feeling lucky" trick if the "ifl" argument is set
    if (err == noError && !args["ifl"].empty()) {
      //Restore the "r" and "o" arg
      args["r"] = starts_old;
      args["o"] = maxDoc_old ;

      //Find whether DocumentSearchResultLinks is enabled
      bool show_links = false;
      text_tmap::const_iterator format_here = cinfo->format.begin();
      text_tmap::const_iterator format_end = cinfo->format.end();

       while (format_here != format_end) {
	 if (((*format_here).first == "DocumentSearchResultLinks") && 
	     ((*format_here).second == "true")){
	   show_links = true;
           break;
	 }
         ++format_here;
       } 

       // Find the search result specified by the ifln argument   
      ResultDocInfo_tarray::iterator thissection = response.docInfo.begin();
      int section_number = 1;
      int ifln = args["ifln"].getint();
              
    while (section_number != ifln  && thissection != response.docInfo.end()) {   	
	thissection++;
        section_number++;
      }
      // If the search result exists, go directly to it
      if (section_number == ifln && thissection != response.docInfo.end()) {
	// Location response (this URL must have "&" and not "&amp;"!)
        int num_docs =  response.numDocs;
        int srn = 0;
	int srp = 0;
        if (show_links && section_number < num_docs ) {
          srn = section_number + 1;
	}    
        if (show_links && section_number > 1 ) {
          srp = section_number - 1;
	}    
        
	textout << outconvert << disp << "Location: _gwcgi_?e=_compressedoptions_&a=d&c=" << collection << "&d=" << (*thissection).OID <<"&srn="<<srn<<"&srp="<<srp<< "\n\n";
	textout << flush;
        return true;
      }
       
      // There weren't enough (or any) matching documents
      // We'll just carry on as if ifl wasn't set. The only catch is that get_cgihead_info won't have
      // done the right thing (because ifl was set), so we need to make sure the output is html
      textout << "Content-type: text/html\n\n";
      
    }

    if (err != noError) {
      disp.setmacro("resultline", "query", "_textnodocs_");
      if (err == syntaxError) {
	disp.setmacro ("freqmsg", "query", "_textinvalidquery_");
      } else {
	disp.setmacro ("freqmsg", "query", "");
      }
    } else {
      
      define_query_macros (args, disp, response.numDocs, response.isApprox);
      define_single_query_macros(args, disp, response);
      // save the query if appropriate
      save_search_history(args, response.numDocs, response.isApprox);
    }

    // If Lucene threw a TooManyClauses exception, tell the user about it
    if (args["ct"] == 2 && response.error_message == "TOO_MANY_CLAUSES") {
      disp.setmacro ("freqmsg", "query", "_textlucenetoomanyclauses_");
    }
  }
  define_history_macros (disp, args, protos, logout);

  textout << outconvert << disp << "_query:header_\n"
	  << "_query:content_";
  
  if (err == noError) {
    // output the results
    text_t numdocs_t = response.numDocs;  
    args["nmd"] = numdocs_t;
    bool use_table = is_table_content (formatlistptr);
    bptr->output_section_group (response, args, collection, 0, formatlistptr, 
				use_table, request.fields, request.getParents, 
				collectproto, disp, outconvert, textout, logout);
  }

  textout << outconvert << disp << "_query:footer_";
  
  delete (formatlistptr);
  
  return true;
}
 
// does the formatting of the query string - either uses q for a text search
// or the form values for an form search
// also adds dates if appropriate in text search
void queryaction::get_formatted_query_string (text_t &formattedstring, 
					      bool segment,
					      cgiargsclass &args, 
					      displayclass &disp,
					      ostream &logout) {
  if (args["qt"]=="0" && args["qto"] != "2") { // normal text search
    formattedstring = args["q"];
    // remove & | ! for simple search,do segmentation if necessary
    format_querystring (formattedstring, args.getintarg("b"), segment);
    if (args["ct"]!=0) { // mgpp and lucene - need to add in tag info if appropriate
      format_field_info(formattedstring, args["fqf"], args.getintarg("ct"), 
			args.getintarg("t"), args.getintarg("b"));
    }

    add_dates(formattedstring, args.getintarg("ds"), args.getintarg("de"),
	      args.getintarg("dsbc"), args.getintarg("debc"),
	      args.getintarg("ct"));
    args["q"] = formattedstring;
    
  }
  else if (args["qt"]=="1" || args["qto"]=="2"){ // form search

    if (args["b"]=="1" && args["fqa"]=="1") { // explicit query
      formattedstring = args["q"];
    }
    else { // form search
      if (args["b"]=="0") { // regular form
	parse_reg_query_form(formattedstring, args, segment);
      }
      else  { // advanced form
	parse_adv_query_form(formattedstring, args, segment);
      }
      args["q"] = formattedstring;
      
      // reset the cgiargfqv macro - need to escape any quotes in it
      disp.setmacro("cgiargfqv", "query", escape_quotes(args["fqv"]));

      // also reset the _cgiargq_ macro as it has changed now
      disp.setmacro("cgiargq", displayclass::defaultpackage, html_safe(args["q"]));
      
      // reset the compressed options to include the q arg
      text_t compressedoptions = recpt->get_compressed_arg(args, logout);
      if (!compressedoptions.empty()) {
	disp.setmacro ("compressedoptions", displayclass::defaultpackage, dm_safe(compressedoptions));
	// need a decoded version of compressedoptions for use within forms
	// as browsers encode values from forms before sending to server
	// (e.g. %25 becomes %2525)
	decode_cgi_arg (compressedoptions); 
	if (args["w"] == "utf-8") { // if the encoding was utf-8, then compressed options was utf-8, and we need unicode.
    // if encoding wasn't utf-8, then compressed opotions may be screwed up, but seems to work for 8 bit encodings?
	  compressedoptions = to_uni(compressedoptions);
	}

	disp.setmacro ("decodedcompressedoptions", displayclass::defaultpackage, dm_safe(compressedoptions));
      }
    } // form search
  } // args["qt"]=1
  else {
    logout << "ERROR (query_action::get_formatted_query_string): querytype not defined\n";
  }
}


// define_query_macros sets the macros that couldn't be set until the
// query had been done. Those macros are 
// _resultline_, _nextfirst_, _nextlast_, _prevfirst_, _prevlast_,
// _thisfirst_, and _thislast_ and _quotedquery_ 
// this has been simplified so it can be used with both search_single_coll
// and search_multiple_coll
void queryaction::define_query_macros (cgiargsclass &args, displayclass &disp,
				       int numdocs, isapprox isApprox) {
  
  // set up _resultline_ macro
  text_t resline;
  int maxdocs = args.getintarg("m");
  if (num_phrases > 0) isApprox = Exact;
  if (maxdocs == -1) maxdocs = numdocs;
  else if (numdocs > maxdocs) {
    numdocs = maxdocs;
    isApprox = MoreThan;
  }

  if (isApprox == Approximate) resline = "_textapprox_";
  else if (isApprox == MoreThan) resline = "_textmorethan_";
  
  if (numdocs == 0) resline = "_textnodocs_";
  else if (numdocs == 1) resline += "_text1doc_";
  else resline += text_t(numdocs) + " _textlotsdocs_";
  
  disp.setmacro("resultline", "query", resline);

  int firstdoc = args.getintarg("r");
  int hitsperpage = args.getintarg("o");
  if (hitsperpage == -1) hitsperpage = numdocs;

  // set up _thisfirst_ and _thislast_ macros
  disp.setmacro ("thisfirst", "query", firstdoc);
  int thislast = firstdoc + (hitsperpage - 1);
  if (thislast > numdocs) thislast = numdocs;
  disp.setmacro ("thislast", "query", thislast);

  // set up _prevfirst_ and _prevlast_ macros
  if (firstdoc > 1) {
    disp.setmacro ("prevlast", "query", firstdoc - 1);
    int prevfirst = firstdoc - hitsperpage;
    if (prevfirst < 1) prevfirst = 1;
    disp.setmacro ("prevfirst", "query", prevfirst);
  }

  // set up _nextfirst_ and _nextlast_ macros
  if (thislast < numdocs) {
    disp.setmacro ("nextfirst", "query", thislast + 1);
    int nextlast = thislast + hitsperpage;
    if (nextlast > numdocs) nextlast = numdocs;
    disp.setmacro ("nextlast", "query", nextlast);
  }

  // do quoted query here cos we may have added quotes during query pre-processing
  if (args["ct"]==0) { // mg queries only, not mgpp
    // get the quoted bits of the query string and set _quotedquery_
    text_tarray phrases;
    get_phrases (args["q"], phrases);
    num_phrases = phrases.size();
    text_tarray::const_iterator phere = phrases.begin();
    text_tarray::const_iterator pend = phrases.end();
    bool first = true;
    text_t quotedquery;
    while (phere != pend) {
      if (!first)
	if ((phere +1) == pend) quotedquery += " and ";
	else quotedquery += ", ";
      
      quotedquery += "\"" + *phere + "\"";
      first = false;
      ++phere;
    }
    if (args.getintarg("s") && !quotedquery.empty()) quotedquery += "_textstemon_";
    disp.setmacro ("quotedquery", "query", quotedquery);
  }

}

// define_single_query_macros sets the extra macros for search_single_coll 
// that couldn't be set until the query had been done. Those macros are 
// _freqmsg_ and _stopwordsmsg_
void queryaction::define_single_query_macros (cgiargsclass &args, 
					      displayclass &disp, 
					      const FilterResponse_t &response) {
  // set up _freqmsg_ and _stopwordsmsg_ macros

  text_t freqmsg = "";
  freqmsg = "_textfreqmsg1_";
  TermInfo_tarray::const_iterator this_term = response.termInfo.begin();
  TermInfo_tarray::const_iterator end_term = response.termInfo.end();
  while (this_term != end_term) {
    freqmsg += (*this_term).term + ": " + (*this_term).freq;
    if ((this_term + 1) != end_term)
      freqmsg += ", ";
    ++this_term;
  }
  disp.setmacro ("freqmsg", "query", freqmsg);

  text_tset::const_iterator this_stopword = response.stopwords.begin();
  text_tset::const_iterator end_stopword = response.stopwords.end();
  if (this_stopword != end_stopword) {
    text_t stopwordsmsg = "_textstopwordsmsg_ ";
    while (this_stopword != end_stopword) {
      if (stopwordsmsg != "_textstopwordsmsg_ ") {
	stopwordsmsg += ", ";
      }
      stopwordsmsg += (*this_stopword);
      ++this_stopword;
    }
    disp.setmacro("stopwordsmsg", "query", stopwordsmsg);
  }
}

// should this change for cross coll search??
bool queryaction::save_search_history (cgiargsclass &args, int numdocs, 
				       isapprox isApprox) {
  if (args["q"]=="") return true; // null query, dont save
  if (args["hs"]=="0") return true; // only save when submit query pressed
  
  // get userid
  text_t userid = args["z"];

  // the number of docs goes on the front of the query string
  text_t query = text_t(numdocs);
  if (isApprox==MoreThan) { // there were more docs found
    query.push_back('+');
  }
  query += "c="+args["c"];
  query += ";h="+args["h"];
  query += ";t="+args["t"];
  query += ";b="+args["b"];
  query += ";j="+args["j"];
  query += ";n="+args["n"];
  query += ";s="+args["s"];
  query += ";k="+args["k"];
  query += ";g="+args["g"]; 
  
  text_t qstring = args["q"];
  //text_t formattedquery =cgi_safe(qstring);
  //query += "&amp;q="+formattedquery;
  query += ";q="+qstring;
  bool display=false;
  int hd = args.getintarg("hd");
  if (hd > 0) display=true;
  if (set_history_info(userid, query, gdbmhome, display)) return true;
  else return false;


}