source: trunk/gsdl/src/library/libinterface.cpp@ 99

#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <time.h>
#include "libinterface.h"
#include "cgiargs.h"
#include "fileutil.h"
#include "cfgread.h"
#include "gsdlunicode.h"
#include "unitool.h"

#include <assert.h>




///////////////////////
// support functions //
///////////////////////

unsigned short hexdigit (unsigned short c)
{
  if (c >= '0' && c <= '9') return (c-'0');
  if (c >= 'a' && c <= 'f') return (c-'a'+10);
  if (c >= 'A' && c <= 'F') return (c-'A'+10);
  return c;
}


void c2hex (unsigned short c, text_t &t)
{
  t.clear();

  if (c >= 256) 
    {
      t = "20"; // ' '
      return;
    }

  unsigned short o1, o2;
 
  o1 = (c/16) % 16;
  o2 = c % 16;
  if (o1 >= 10) o1 += 'a' - 10;
  else o1 += '0';
  if (o2 >= 10) o2 += 'a' - 10;
  else o2 += '0';

  t.push_back(o1);
  t.push_back(o2);
}

// convert %xx and + to their appropriate equivalents
void decode (text_t &argstr)
{
  text_t::iterator in = argstr.begin();
  text_t::iterator out = in;
  text_t::iterator end = argstr.end();

  while (in != end)
    {
      if (*in == '+') *out = ' ';

      else if (*in == '%')
    {
      unsigned short c = '%';
      in++;
      if (in != end)
        {
          c = hexdigit (*in);
          in++;
        }
      if (in != end && c < 16) // sanity check on the previous character
        {
          c = c*16 + hexdigit (*in);
        }

      *out = c;
    }
      else *out = *in;

      if (in != end) in++;
      out++;
    }
  
  // remove the excess characters
  argstr.erase (out, end);
}


// split up the cgi arguments
void parse_cgi_args (text_t argstr, cgiargsclass &args)
{
  args.clear();

  text_t::iterator here = argstr.begin();
  text_t::iterator end = argstr.end();
  text_t key, value;

  // extract out the key=value pairs
  while (here != end) 
    {
      // get the next key and value pair
      here = getdelimitstr (here, end, '=', key);
      here = getdelimitstr (here, end, '&', value);

      // convert %xx and + to their appropriate equivalents
      decode (value);
      value.setencoding(1); // other encoding
      // store this key=value pair
      if (!key.empty()) args.setarg (key, value);
    }
}

text_t cgisafe (const text_t &intext)
{
  text_t outtext;

  text_t::const_iterator here = intext.begin ();
  text_t::const_iterator end = intext.end ();
  unsigned short c;
  text_t ttmp;

  while (here != end)
    {
      c = *here;
      if (((c >= 'a') && (c <= 'z')) ||
      ((c >= 'A') && (c <= 'Z')) ||
      ((c >= '0') && (c <= '9'))) {
    // alphanumeric character
    outtext.push_back(c);
      } else {
      // non-alphnumeric character
      outtext.push_back('%');
      c2hex(c, ttmp);
      outtext += ttmp;
      }
      
      here++;
    }

  return outtext;
}

//////////////////////////////
// methods for libinterface //
//////////////////////////////

// constructor

libinterface::libinterface() {
  browse = NULL;
}

void libinterface::setgsdlhome (const text_t &thegsdlhome) {
  text_t thecollectdir, thegdbmdir, filename;

  gsdlhome = thegsdlhome;

  // search for etc/collect.cfg
  thecollectdir = filename_cat (gsdlhome, "collect");
  thecollectdir = filename_cat (thecollectdir, get_collection_name());
  filename = filename_cat (thecollectdir, "etc");
  filename = filename_cat (filename, "collect.cfg");

  if (!file_exists(filename)) thecollectdir = gsdlhome;

  thegdbmdir = filename_cat (thecollectdir, "index");
  thegdbmdir = filename_cat (thegdbmdir, "text");

  setcollectdir (thecollectdir);
  setgdbmdir (thegdbmdir);
}

void libinterface::setcollectdir (const text_t &thecollectdir) {
  collectdir = thecollectdir;

  search.setcollectdir (collectdir);
}

void libinterface::setgdbmdir (const text_t &thegdbmdir) {
  gdbmdir = thegdbmdir;
}

void libinterface::sethttpprefix (const text_t &thehttpprefix) {
  httpprefix = thehttpprefix;
}

void libinterface::setgwcgi (const text_t &thegwcgi) {
  gwcgi = thegwcgi;
}


int libinterface::cfg_read (const text_t &filename) {
  text_t key;
  text_tarray cfgline;
  char *cstr = filename.getcstr();
  ifstream confin (cstr);
  delete cstr;

  if (confin) {
    while (read_cfg_line(confin, cfgline) >= 0) {
      if (cfgline.size () >= 2) {
    key = cfgline[0];
    cfgline.erase(cfgline.begin());
    if (key == "maintainer")        cfg_info.maintainer = cfgline[0];
    else if (key == "indexes")      cfg_info.indexes = cfgline;
    else if (key == "defaultindex") cfg_info.defaultindex = cfgline[0];
    else if (key == "macrofiles")   cfg_info.macrofiles = cfgline;
    else if (key == "builddate")    cfg_info.builddate = cfgline[0];
    else if (key == "indexmap")     cfg_info.indexmap = cfgline;
    else if (key == "numbytes")     cfg_info.numbytes = (double)cfgline[0].getint();
    else if (key == "numdocs")      cfg_info.numdocs = (double)cfgline[0].getint();
    else if (key == "defaultencoding") 
      cfg_info.defaultencoding = cfgline[0];
      }
    }
    confin.close ();
    return 1;
  }
  return 0;
}


// init should be called after the various homes are set,
// it returns 'false' on failure and 'true' on success
bool libinterface::init (ostream &logout) {
  text_t collection = get_collection_name();

  // redirect the error output to logout
  disp.setlogout (&logout);
  gdbm.setlogout (&logout);

  // open the gdbm file
  text_t filename = filename_cat (gdbmdir, get_collection_name ());
#ifdef _LITTLE_ENDIAN
  filename += ".ldb"; // little endian version of the gdbm database
#else
  filename += ".bdb"; // big endian version on the gdbm database
#endif
  gdbm.opendatabase (filename);


  // set default values for the configuration file
  cfg_info.defaultencoding = "w";

  // read in the configuration files etc/collect.cfg and index/build.cfg
  // entries in build.cfg should override those in collect.cfg
  filename = filename_cat (collectdir, "etc");
  filename = filename_cat (filename, "collect.cfg");
  cfg_read(filename);
  filename = filename_cat (collectdir, "index");
  filename = filename_cat (filename, "build.cfg");
  cfg_read(filename);

  //  logout << logconvert << "defaultindex: " << cfg_info.defaultindex << "\n";

  // set the default index
  if (cfg_info.indexmap.empty()) {
    // ?? no indexes built ??
    logout << "warning: no indexes have been built\n";
    default_index.clear();
  } else if (cfg_info.defaultindex.empty() || 
         !isrealindex (cfg_info.indexmap, cfg_info.defaultindex)) {
    logout << "warning: the default index has been reset to the first index\n";
    getrealdirindex (cfg_info.indexmap[0], cfg_info.defaultindex, default_index);
  } else {
    default_index = real2dirindex (cfg_info.indexmap, cfg_info.defaultindex);
  }

  // set the text default index (the default index to use when
  // retrieving documents).
  text_default_index = default_index;
  if (!isdoclevelindex (cfg_info.defaultindex)) {
    text_default_index = real2dirindex (cfg_info.indexmap, 
                    getdoclevelindex (cfg_info.indexmap));
  }

  // load up the default macro files, the collection directory
  // is searched first for the file and then the main directory
  text_t colmacrodir = filename_cat (collectdir, "macros");
  text_t gsdlmacrodir = filename_cat (gsdlhome, "macros");
  text_tarray::iterator arrhere = cfg_info.macrofiles.begin();
  text_tarray::iterator arrend = cfg_info.macrofiles.end();
  while (arrhere != arrend) {
    filename = filename_cat (colmacrodir, *arrhere);
    if (!file_exists (filename)) {
      filename = filename_cat (gsdlmacrodir, *arrhere);
    }
    disp.loaddefaultmacros(filename);
    arrhere++;
  }

  srand(time(NULL));

  utf8outconvert.set_rzws(1);
  gboutconvert.set_rzws(1);

  return collection_init(collection);
}


// examine the cgi arguments and create the appropriate page, 
// outputing the page to textout and any debug information to logout
//
// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
int libinterface::getpage (const text_t &argstr, ostream &textout, ostream &logout) {
  int err = LI_NOERROR;

  // make the output go where they want!
  gdbm.setlogout (&logout);
  disp.setlogout (&logout);

  cgiargsclass args;

  parse_cgi_args (argstr, args);
  expand_compressed_args (args);
  add_default_args (args);
  check_args (args);

  // get the input encoding
  text_t &arg_w = args["w"];
  inconvertclass *inconvert = NULL;
  if (arg_w == "8") {
    inconvert = &utf8inconvert;
  } else if (arg_w == "g") {
    // The map files will only be loaded the first time they are
    // needed. The loading is done here to reduce the memory load
    // for collections which don't need to convert to GB.
    gbinconvert.loadmapfile (gsdlhome, "gbku", 0x25a1);
    inconvert = &gbinconvert;
  } else {
    inconvert = &asciiinconvert; // default
  }

  // see if the next page will have a different encoding
  if (args.getarg("nw") != NULL) args["w"] = args["nw"];

  // convert arguments which aren't in unicode to unicode
  args_tounicode (args, *inconvert);

  // remember the state of the compressed arguments
  lastcomparg = get_compressed_args (args);
  logout << args;

  // get the output encoding
  text_t &arg_nw = args["w"];
  outconvertclass *outconvert = NULL;
  if (arg_nw == "8") {
    outconvert = &utf8outconvert;
  } else if (arg_nw == "g") {
    gboutconvert.loadmapfile (gsdlhome, "ugbk", 0xa1f5);
    outconvert = &gboutconvert;
  } else {
    outconvert = &asciioutconvert; // default
  }


  // dispatch the request
  text_t &arg_a = args["a"];
  if (arg_a == "q") err = query_action (args, *outconvert, textout, logout);
  else if (arg_a == "b") browse_action (args, *outconvert, textout, logout);
  else if (arg_a == "t") document_action (args, *outconvert, textout, logout);
  else if (arg_a == "p") page_action (args, *outconvert, textout, logout);
  else if ((arg_a.size() == 2) && (arg_a[0] == 'a')) 
    auxiliary_action (args, *outconvert, textout, logout);
  else 
    {
      // output error page
    }

  return err;
}

// the arg config string is used to do processing on the arguments
// entries take the form argname[defaultvalue]
// if the argument name is preceeded by a "+" it means that the
// value may be more than one character long
// the main state variable missed on this list is "q" the query string
void libinterface::get_arg_config (text_t &argconfigstr)
{
  argconfigstr = 
    "+a[p]"    // action: q=query, b=browse, t=targetdoc, p=page, a1=auxiliary
    "w[]"      // encoding: w=western, 8=utf8, 7=utf7, g=GB2312, k=GBK
    "t[1]"     // query type: 0=boolean, 1=ranked
    "+i[]"      // index: collection dependant
    "k[1]"     // casefolding: 0=off, 1=on
    "s[0]"     // stemming: 0=off, 1=on
    "+p[home]" // page
    "+c[]"     // collection (collection dependant)
    "+r[1]"    // results from
    "+d[C.1]"  // the target document
    "+j[11]"   // partial index: 11=all, 10=f&n, 01=other
    "+m[100]"  // maxdocs
    "+o[20]"   // hits per page
    "v[0]"     // version: 0=text+graphics, 1=text
    "f[0]"     // query box size: 0=normal, 1=big
    "l[e]";    // language: e=english, m=maori
  
}

text_t::iterator libinterface::get_next_config_arg (text_t::iterator first, 
                            text_t::iterator last,
                            text_t &argname, 
                            text_t &defaultvalue, 
                            bool &longarg)
{
  first = getdelimitstr (first, last, '[', argname);
  first = getdelimitstr (first, last, ']', defaultvalue);
  longarg = false;

  if (!argname.empty() && (argname[0] == '+'))
    {
      argname.erase(argname.begin(), argname.begin()+1);
      longarg = true;
    }

  return first;
}

text_t libinterface::get_compressed_args (cgiargsclass &args)
{
  text_t argconfigstr; get_arg_config (argconfigstr);
  text_t arg_e;
  text_t argname, defaultvalue;
  text_t *argvalue;
  bool longarg;

  text_t::iterator here = argconfigstr.begin();
  text_t::iterator end = argconfigstr.end();
  while (here != end)
    {
      here = get_next_config_arg (here, end, argname, defaultvalue, longarg);

      if (!argname.empty())
    {
      argvalue = args.getarg (argname);
      if (argvalue == NULL) arg_e += defaultvalue;
      else arg_e += *argvalue;
        
      if (longarg) arg_e += "-";
    }
    }
  
  return arg_e;
}


// the compressed options should never override explicit options
// but they should always be expanded before add_default_args is
// called
void libinterface::expand_compressed_args (cgiargsclass &args)
{
  text_t *arg_e = args.getarg("e");

  // see if there is compressed options
  if (arg_e != NULL)
    {
      text_t argconfigstr; get_arg_config (argconfigstr);
      text_t argname, defaultvalue, argvalue;
      bool longarg;

      text_t::iterator confighere = argconfigstr.begin();
      text_t::iterator configend = argconfigstr.end();

      text_t::iterator arghere = arg_e->begin();
      text_t::iterator argend = arg_e->end();
      while (confighere != configend && arghere != argend)
    {
      confighere = get_next_config_arg (confighere, configend, argname, 
                        defaultvalue, longarg);
      if (!argname.empty())
        {
          if (longarg)
        {
          arghere = getdelimitstr (arghere, argend, '-', argvalue);
          if (!argvalue.empty()) args.setdefaultarg (argname, argvalue);
        }
          else
        {
          args.setdefaultcarg (argname,*arghere);
          arghere++;
        }
        }
    }
    }
}


void libinterface::add_default_args (cgiargsclass &args)
{ 
  text_t argconfigstr; get_arg_config (argconfigstr);
  text_t argname, defaultvalue;
  bool longarg;

  text_t::iterator confighere = argconfigstr.begin();
  text_t::iterator configend = argconfigstr.end();
  while (confighere != configend)
    {
      confighere = get_next_config_arg (confighere, configend, argname, 
                    defaultvalue, longarg);
      if (!argname.empty()) args.setdefaultarg (argname, defaultvalue);
    }
  
  // the query string and format string are not included in the argument configuration string
  args.setdefaultarg ("q", ""); // the default query string is ""
  args.setdefaultarg ("g", "00");
  args.setdefaultarg ("x", "0");
}


// check and attempt to fix an problems encountered in the list
// of cgi arguments
void libinterface::check_args (cgiargsclass &args)
{
  args.setarg("c", get_collection_name());
  if (args["w"].empty()) args.setarg("w", cfg_info.defaultencoding);
  if (args["i"].empty()) args.setarg("i", default_index);
}

void libinterface::args_tounicode (cgiargsclass &args, 
                   inconvertclass &inconvert) {
  utf8outconvertclass text_t2utf8;
  cgiargsclass::iterator here = args.begin();
  cgiargsclass::iterator end = args.end();

  while (here != end) {
    if (here->second.getencoding() > 0) {
      here->second = inconvert.convert(here->second);
    }
    
    here++;
  }
}


void libinterface::define_general_macros (cgiargsclass &args, outconvertclass &outconvert, 
                      ostream &logout) {
  disp.setmacro("httpprefix", "Global", httpprefix);
  disp.setmacro("gwcgi", "Global", gwcgi);

  disp.setmacro("numdocs", "Global", (int)cfg_info.numdocs);

  disp.setmacro("collection", "Global", cgisafe(outconvert.convert(args["c"])));
  disp.setmacro("compressedoptions", "Global", get_compressed_args(args));
  disp.setmacro("urlsafequerystring", "Global", cgisafe(outconvert.convert(args["q"])));

  // need to escape any special characters in querystring to prevent
  // them upsetting the html
  text_t querystring;
  text_t::iterator here = args["q"].begin();
  text_t::iterator end = args["q"].end();
  while (here != end) {
    if (*here == '"') querystring += "&quot;";
    else if (*here == '&') querystring += "&amp;";
    else if (*here == '<') querystring += "&lt;";
    else if (*here == '>') querystring += "&gt;";
    else querystring.push_back(*here);
    here ++;
  }
  disp.setmacro("querystring", "Global", querystring);

  if (args.getintarg("x") == 0) disp.setmacro("notdetached", "Global", "1");
  if (args["d"][0] == 'T') disp.setmacro("istitle", "Global", "1");

  int i = rand();
  disp.setmacro("pagedest", "Global", text_t(i));

  // define the macro for the "g" argument
  disp.setmacro("g", "Global", args["g"]);

 // set the selection macros

  text_t indexselect;
  text_t maprealindex, mapdirindex;
  if (cfg_info.indexmap.size() == 1) {
    getrealdirindex (cfg_info.indexmap[0], maprealindex, mapdirindex);
    indexselect += "<input type=hidden name=\"i\" value=\"";
    indexselect += mapdirindex;
    indexselect += "\">_query:";
    indexselect += real2macroindex (maprealindex);
    indexselect += "_\n";

  } else {
    text_t &arg_i = args["i"];
    text_tarray::const_iterator maphere = cfg_info.indexmap.begin();
    text_tarray::const_iterator mapend = cfg_info.indexmap.end();
    
    indexselect += "<select name=\"i\">\n";
    while (maphere != mapend) {
      getrealdirindex (*maphere, maprealindex, mapdirindex);
      indexselect += "<option value=\"";
      indexselect += mapdirindex;
      indexselect += "\"";
      if (arg_i == mapdirindex) indexselect += " selected";
      indexselect += ">_query:";
      indexselect += real2macroindex (maprealindex);
      indexselect += "_\n";
      
      maphere++;
    }
    indexselect += "</select>\n";
  }

  disp.setmacro("indexselection", "query", indexselect);

  text_t qtselect;
  text_t &arg_t = args["t"];

  qtselect += "<select name=\"t\">\n";
  qtselect += "<option value=\"1\"";
  if (arg_t == "1") qtselect += " selected";
  qtselect += ">_query:textsome_\n";
  qtselect += "<option value=\"0\"";
  if (arg_t == "0") qtselect += " selected";
  qtselect += ">_query:textall_\n";
  qtselect += "</select>\n";

  disp.setmacro("querytypeselection", "query", qtselect);
}


// prepare_page prepares to write out a page using the current 
// page parameters and defines any general macros
void libinterface::prepare_page (cgiargsclass &args, outconvertclass &outconvert,
                 ostream &logout) {
  // get page parameters
  text_t pageparams = text_t("collection=") + args["c"];
  if (args.getintarg("u") == 1) pageparams += ",style=htmlonly";
  if (args.getintarg("v") == 1) pageparams += ",version=text";
  if (args.getintarg("f") == 1) pageparams += ",queryversion=big";
  if (args["l"] != 'e') pageparams += ",language=" + args["l"];

  disp.openpage(pageparams, MACROPRECEDENCE);
  define_general_macros(args, outconvert, logout);
  define_collection_macros(args, logout);
}

void libinterface::set_query_params (cgiargsclass &args, queryparamclass &queryparams)
{
  queryparams.collection = args["c"];
  //  assemble_index (args, queryparams.search_index);
  queryparams.search_index = args["i"];
  queryparams.querystring = args["q"];
  format_querystring (queryparams.querystring);
  queryparams.search_type = args.getintarg ("t");
  queryparams.casefolding = args.getintarg ("k");
  queryparams.stemming = args.getintarg ("s");
  queryparams.maxdocs = args.getintarg ("m");
}

void libinterface::format_querystring (text_t &querystring)
{
  text_t formattedstring;
  quotedstring.clear();

  text_t::iterator here = querystring.begin();
  text_t::iterator end = querystring.end();
  int foundquote = 0;

  // want to remove ()|!& from querystring so boolean queries are just
  // "all the words" queries
  while (here != end) {
    if (*here == '(' || *here == ')' || *here == '|' ||
    *here == '!' || *here == '&') {
      formattedstring += " ";
    } else {
      if (*here == '"') {
    if (foundquote) {foundquote = 0; quotedstring.push_back(*here);}
    else foundquote = 1;
      } else {
    formattedstring.push_back(*here);
      }
      if (foundquote) quotedstring.push_back(*here);
    }
    here ++;
  }
  querystring = formattedstring + quotedstring;
}

void libinterface::define_query_macros (cgiargsclass &args, 
                    queryparamclass &queryparams,
                    queryresultsclass &queryresults, 
                    ostream &logout) 
{
  int numdocs = queryresults.getnumdocs();
  int numterms = queryresults.getnumterms();
  disp.setmacro("querysize", "query", args["f"]);
  disp.setmacro("haveresults", "query", numdocs);
  
  // set the display frequency macro
  text_t freqmsg = "_textfm1_";

  int first = 1;
  for (int i = 0; i < numterms; i++) {
    if (first == 0) freqmsg += "; ";
    first = 0;
    freqmsg += queryresults.terms[i].termstr + ": " + queryresults.terms[i].termfreq;
  }
  if (!quotedstring.empty()) freqmsg += "<br><i>post-processed to find " + quotedstring + "</i>\n";
  disp.setmacro("freqmsg", "query", freqmsg);

  // set the result line macro

  text_t resline;
  
  if (numdocs >= queryparams.maxdocs)
    resline.setcstr("_textmt2_");
  
  if (numdocs == 0) {
    resline.setcstr("_textndmtq_");
  } else if (numdocs == 1) {
    resline += text_t(numdocs) + text_t(" _textdmtq2_.");
  } else {
    resline += text_t(numdocs) + text_t(" _textdmtq3_.");
  }

  disp.setmacro("resultline", "query", resline);

  //  define_collection_macros (args, logout);

  if (queryresults.getnumdocs() > 0) {
    docLinks(args, queryresults, logout);
  }
}


// set the _links_ macro to create the links between pages of query results
void libinterface::docLinks (cgiargsclass &args, 
                 queryresultsclass &queryresults, 
                 ostream &logout) 
{
  text_t links;
  int a, b, documents, nextfirst, nextlast, prevfirst, prevlast;
  int results_from = args.getintarg("r");
  int hitsperpage = args.getintarg("o");
  
  documents = queryresults.getnumdocs();
  
  a = results_from;
  b = a + (hitsperpage - 1);
  
  // make sure a and b are in range
  if (a < 1) a = 1;
  if (b < 1) b = 1;
  if (a > documents) a = documents;
  if (b > documents) b = documents;
  
  links.setcstr("<table cellspacing=0 cellpadding=0 border=0 width=\"100%\">\n");
  links += "<tr valign=bottom>\n";
  links += "<td align=left>\n";
  links += "<_font_>\n";
  
  // previous page link
  if (a > 1) {
    prevlast = a - 1;
    prevfirst = a - hitsperpage;
    
    links += "<p><a href=\"_httpqueryresults_&r="; 
    links += prevfirst;
    links += "\">_iconprev__textmatches_ ";
    links += prevfirst;
    links += " - ";
    links += prevlast;
    links += "</a>\n";
  }
  
  links += "</font>\n";
  links += "</td><td width=100></td><td align=right>\n";
  links += "<_font_>\n";
  
  // next page link
  if (b < documents) {
    nextfirst = b + 1;
    nextlast = b + hitsperpage;
    if (nextlast > documents) nextlast = documents;
    
    links += "<p><a href=\"_httpqueryresults_&r=";
    links += nextfirst;
    links += "\">_textmatches_ ";
    links += nextfirst;
    links += " - ";
    links += nextlast ;
    links += "_iconnext_</a>\n";
  }
  
  links += "</font>\n";
  links += "</td></tr></table>\n";
  
  disp.setmacro("links", "query", links);
}


// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
int libinterface::do_query(cgiargsclass &args, queryparamclass &queryparams, 
                queryresultsclass &queryresults, ostream &logout) 
{
  set_query_params(args, queryparams);

  if (!queryparams.querystring.empty()) {
    // do the query - the results are returned in queryresults
    if (!search.search(queryparams, queryresults)) {
      logout << "ERROR: database didn't load\n";
      return LI_LOADDATABASEFAILED;
    }
  }
  return LI_NOERROR;
}

////////////////////////////////////////////////////////////////////////////////////////
// query_action is called whenever a search is to be carried out  (i.e. when the 
// 'a' parameter == 'q') - query calls the mgsearch search() function (via do_query()) to 
// carry out the search then displays the first page of results.
//
// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
int libinterface::query_action (cgiargsclass &args, outconvertclass &outconvert, 
                ostream &textout, ostream &logout) {
  int err = LI_NOERROR;

  queryparamclass queryparams;
  queryresultsclass queryresults;

  err = do_query(args, queryparams, queryresults, logout);

  // prepare to print out the page
  prepare_page(args, outconvert, logout);
  define_query_macros(args, queryparams, queryresults, logout);

  // print out the query page
  textout << outconvert << disp << "_query:header_\n";

  // output query results if there is a query string - 
  // otherwise output help text
  if (!queryparams.querystring.empty()) 
    {
      displayresults (args, outconvert, textout, logout, queryresults);
    } 
  else 
    {
      textout << outconvert << disp << "_query:noqueryheader_\n";
    }
  
  textout << outconvert << disp << "_query:footer_\n";

  return err;
}

void libinterface::displayresults (cgiargsclass &args, outconvertclass &outconvert, 
                   ostream &textout, ostream &logout,
                   queryresultsclass &queryresults) {
  textout << outconvert << disp << "_query:queryheader_";

  int startresults = args.getintarg("r") - 1;
  int numresults = args.getintarg("o");

  textout << outconvert << "<table cellspacing=4>\n";
  for (int i=startresults; i < startresults+numresults; i++) {
    displaydocsummary (args, outconvert, textout, logout, queryresults, i);
  }
  textout << outconvert << "</table>\n\n";

  textout << outconvert << disp << "_query:queryfooter_";
}


///////////////////////////////////////////////////////////////////////////////////////////////
// browse_action writes out the browse pages (i.e. the top level hierarchy pages)
void libinterface::browse_action (cgiargsclass &args, outconvertclass &outconvert, 
                  ostream &textout, ostream &logout) {
  text_t browse_bar, locator, output;
  gdbm_info info;
  
  prepare_page(args, outconvert, logout);
  
  // get browse bar unless page has been detached 
  if (args.getintarg("x") == 0) {
    browse->get_browse_bar(args["d"], browse_bar);
  }
  
  // get top locator
  browse->get_top_locator(args, gdbm, 0, locator);  

  // expand and output page 
  // Note: we need to expand these out using package 'browse'
  // so we can't use the tricky '<<' syntax
  disp.expandstring("browse", "_header_", output);
  textout << outconvert << output;
  disp.expandstring("browse", browse_bar, output);
  textout << outconvert << output;
  disp.expandstring("browse", locator, output);
  textout << outconvert << output;
  disp.expandstring("browse", "_footer_", output); 
  textout << outconvert << output;
}



/////////////////////////////////////////////////////////////////////////////////////////
// document_action is called to retrieve and display collection documents
// It calls the mgsearch function docTargetDocument() to retrieve 
// a document. 

void libinterface::document_action (cgiargsclass &args, outconvertclass &outconvert, 
                    ostream &textout, ostream &logout) {
  
  text_t locator, content, links, output;
  gdbm_info info;
  queryparamclass queryparams;
  queryresultsclass queryresults;
  int oversize = 0;

  // have to redo the query to get queryterms for highlight text
  do_query(args, queryparams, queryresults, logout);

  prepare_page(args, outconvert, logout);
  
  if (args["g"][1] == '0') {
    // get docnum from gdbm
    text_t docref;
    if (args["d"][0] != 'B') get_book(args["d"], docref);
    else docref = args["d"];
    if (gdbm.getinfo(docref, info) != 0) {
      logout << logconvert << "info_db wasn't opened - " << docref << "\n";
      return;
    }
    
    // get document text if there is any
    if (info.contents.empty()) {
        search.docTargetDocument(text_default_index, queryparams.collection,
                 info.docnum, content);
    if (info.title != "<i>(introductory text)</i>")
      content = "<h3>" + info.title + "</h3>\n" + content;
    }
  } 

  if (args["g"][1] == '1') {
    // want to get expanded out text
    vector<text_t> contents_arr;
    text_t booksection;
    int levelcount;

    get_book (args["d"], booksection);
    levelcount = count_dots(booksection);

    browse->get_contents_arr(args, gdbm, contents_arr);
    

    // get text for each section of book
    vector<text_t>::const_iterator thiscontent = contents_arr.begin();
    vector<text_t>::const_iterator end = contents_arr.end();

    int first = 1;
    int count = 1;
    while (thiscontent != end) {
      text_t text;

      // get docnum from gdbm
      if (gdbm.getinfo(*thiscontent, info) != 0) { 
    logout << logconvert << "info_db wasn't opened - " << args["d"] << "\n";
    return;
      }

      // if section has text get it, otherwise output section title
      if (info.contents.empty()) {

    // output <a name= > tags for all text sections currently displayed in toc (all text sections
    // if contents are expanded
    if (count_dots(*thiscontent) == levelcount || args["g"][0] == '1') {
      content += "<a name=\"";
      content += count;
      content += "\"></a>\n";
      count ++;
    }

    search.docTargetDocument(text_default_index, queryparams.collection,
                 info.docnum, text);                    
    if (info.title != "<i>(introductory text)</i>") 
      content += "<h3>" + info.title + "</h3>\n";
    //  content += text + "<hr><br>\n";
    content += text + "<p>\n"; // no longer want <hr> between sections
      } else {
    content += "<h3>" + info.title + "</h3>\n";
      }

      if (args["n"] == 1) {
    if (first) {
      browse->get_top_locator(args, gdbm, 0, locator);
      disp.expandstring("text", "_header_", output);
      textout << outconvert << output;
      disp.expandstring("text", locator, output);
      textout << outconvert << output;
    }
    
    disp.expandstring("text", content, output);
    if (!queryparams.querystring.empty())
      highlighttext(queryresults.termvariants, outconvert, textout, logout, output);
    else
      textout << outconvert << output;
    first = 0;
    content.clear();
      }

      thiscontent ++;
      if (content.size() > 200000 && args["n"] != 1) {
    content.clear();
    oversize = 1;
    args["g"][1] = '0';
    break;
      }
    }

    if (args["g"][1] == '0') {
      // get docnum from gdbm
      text_t docref;
      if (args["d"][0] != 'B') get_book(args["d"], docref);
      else docref = args["d"];
      if (gdbm.getinfo(docref, info) != 0) {
        logout << logconvert << "info_db wasn't opened - " << docref << "\n";
        return;
      }
    
      // get document text if there is any
      if (info.contents.empty()) {
        search.docTargetDocument(text_default_index, queryparams.collection,
                                 info.docnum, content);
    if (info.title != "<i>(introductory text)</i>")
      content = "<h3>" + info.title + "</h3>\n" + content;
      }
    }
  }

  if (args["n"] != 1) {
    // get top locator
    browse->get_top_locator(args, gdbm, oversize, locator);

    // expand and output page 
    // Note: we need to expand these out using package 'text'
    // so we can't use the tricky '<<' syntax
    disp.expandstring("text", "_header_", output);
    textout << outconvert << output;

    disp.expandstring("text", locator, output);
    textout << outconvert << output;

    disp.expandstring("text", content, output);
    if (!queryparams.querystring.empty())
      highlighttext(queryresults.termvariants, outconvert, textout, logout, output);
    else
      textout << outconvert << output;
  }

  // get links to next and previous sections unless in expand text mode
  if (args["g"][1] == '0') {
    browse->get_links(args, gdbm, links);
    disp.expandstring("text", links, output);
    textout << outconvert << output;
  }

  disp.expandstring("text", "_footer_", output); 
  textout << outconvert << output;
}


/////////////////////////////////////////////////////////////////////////////////////////
// auxiliary_action is called to retrieve and display collection documents
// in formats other than those handled by document_action (i.e. those other
// than text). This should be overridden for collections needing to return
// images, postscript etc. You can have as many auxiliary actions as needed
// by setting arg_a to a1, a2, a3 etc. and testing arg["a"][1] within the 
// auxiliary_action function.
// auxiliary_action defaults to calling document_action

void libinterface::auxiliary_action (cgiargsclass &args, outconvertclass &outconvert, 
                     ostream &textout, ostream &logout) {
  document_action (args, outconvert, textout, logout);
}

/////////////////////////////////////////////////////////////////////////////////////////
// page is called when a standard html page is to be displayed
void libinterface::page_action (cgiargsclass &args, outconvertclass &outconvert, 
                ostream &textout, ostream &logout) {

  text_t &arg_p = args["p"];

  prepare_page(args, outconvert, logout);

  if (arg_p == "preferences")
    define_pref_macros(args, logout);

  textout << outconvert << disp << ("_" + arg_p + ":header_\n")
      << ("_" + arg_p + ":imagestandardbar_\n") << ("_" + arg_p + ":content_\n")
      << ("_" + arg_p + ":footer_\n");
}

// highlighttext highlights query terms in text string and outputs the resulting text string
void libinterface::highlighttext(text_tarray &termvars, outconvertclass &outconvert, 
                 ostream &textout, ostream &logout, text_t &text) {
  map<text_t, int, lttext_t> terms;
  map<text_t, int, lttext_t>::const_iterator it;
  for (unsigned int i = 0; i < termvars.size(); i++) {
    terms[termvars[i]] = 1;
  }

  text_t::iterator here = text.begin(); 
  text_t::iterator end = text.end();
  text_t word, buffer;
  while (here != end) {
    if (is_unicode_letdig(*here)) {
      // not word boundary
      word.push_back(*here);
      here++;

    } else {
      // found word boundary
      // add last word if there was one
      if (!word.empty()) {
    it = terms.find(word);
    if (it != terms.end()) {
      word = "<b><u>" + word + "</u></b>";
    }
    buffer += word;
        word.clear();
      }

      if (*here == '<') {
        // skip over rest of html tag
    while ((here != end) && (*here != '>')) {
      buffer.push_back(*here);
      here++;
    }
      }

      buffer.push_back(*here);
      here++;

      if (buffer.size() > 1024) {
    textout << outconvert << buffer;
    buffer.clear();
      }
    }
  }
  textout << outconvert << buffer;
}

void libinterface::define_pref_macros (cgiargsclass &args, ostream &logout)
{
  // the caseoption macro
  text_t caseoption;
  int arg_k = args.getintarg("k");

  caseoption += "\n<input type=radio name=k value=1";
  if (arg_k) caseoption += " checked";
  caseoption += "> ignore case differences<br>\n";
  caseoption += "<input type=radio name=k value=0";
  if (!arg_k) caseoption += " checked";
  caseoption += "> upper/lower case must match\n";

  disp.setmacro ("caseoption", "preferences", caseoption);

  // the stemoption macro
  text_t stemoption;
  int arg_s = args.getintarg("s");

  stemoption += "\n<input type=radio name=s value=1";
  if (arg_s) stemoption += " checked";
  stemoption += "> ignore word endings<br>\n";
  stemoption += "<input type=radio name=s value=0";
  if (!arg_s) stemoption += " checked";
  stemoption += "> whole word must match\n";

  disp.setmacro ("stemoption", "preferences", stemoption);


  // the encodingoption
  text_t encodingoption;
  const text_t &arg_w = args["w"];

  encodingoption += "\n<select name=\"nw\">\n";
  encodingoption += "  <option value=\"w\"";
  if (arg_w == "w") encodingoption += " selected";
  encodingoption += ">Western (ISO-8859-1)\n";
  encodingoption += "  <option value=\"g\"";
  if (arg_w == "g") encodingoption += " selected";
  encodingoption += ">Simplified Chinese (GB2312)\n";
  encodingoption += "  <option value=\"8\"";
  if (arg_w == "8") encodingoption += " selected";
  encodingoption += ">Unicode (UTF-8)\n";
  encodingoption += "</select>\n";

  disp.setmacro ("encodingoption", "preferences", encodingoption);

  // the maxdocoption
  text_t maxdocoption;
  int arg_m = args.getintarg("m");

  maxdocoption += "\n<select name=m>\n";
  maxdocoption += "  <option value=\"50\"";
  if (arg_m < 100) maxdocoption += " selected";
  maxdocoption += ">50\n";
  maxdocoption += "  <option value=\"100\"";
  if (arg_m >= 100 && arg_m < 200) maxdocoption += " selected";
  maxdocoption += ">100\n";
  maxdocoption += "  <option value=\"200\"";
  if (arg_m >= 200 && arg_m < 500) maxdocoption += " selected";
  maxdocoption += ">200\n";
  maxdocoption += "  <option value=\"500\"";
  if (arg_m >= 500) maxdocoption += " selected";
  maxdocoption += ">500\n";
  maxdocoption += "</select>\n";

  disp.setmacro ("maxdocoption", "preferences", maxdocoption);

  // the hitsperpageoption
  text_t hitsoption;
  int arg_o = args.getintarg("o");

  hitsoption += "\n<select name=o>\n";
  hitsoption += "  <option value=\"10\"";
  if (arg_o < 20) hitsoption += " selected";
  hitsoption += ">10\n";
  hitsoption += "  <option value=\"20\"";
  if (arg_o >= 20 && arg_o < 50) hitsoption += " selected";
  hitsoption += ">20\n";
  hitsoption += "  <option value=\"50\"";
  if (arg_o >= 50 && arg_o < 100) hitsoption += " selected";
  hitsoption += ">50\n";
  hitsoption += "  <option value=\"100\"";
  if (arg_o >= 100 && arg_o < 500) hitsoption += " selected";
  hitsoption += ">100\n";
  hitsoption += "  <option value=\"500\"";
  if (arg_o >= 500) hitsoption += " selected";
  hitsoption += ">all\n";
  hitsoption += "  </select>\n";

  disp.setmacro ("hitsperpageoption", "preferences", hitsoption);
}