source: trunk/gsdl/lib/display.cpp@ 1270

/**********************************************************************
 *
 * display.cpp -- Context sensitive macro language
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * $Id: display.cpp 1217 2000-06-18 22:56:55Z sjboddie $
 *
 *********************************************************************/

/*
   $Log$
   Revision 1.21  2000/06/18 22:56:55  sjboddie
   fixed a bug that I'd introduced earlier when attempting to get things
   compiling on VC++ 6.0 - the gsdl-2.22 release (and possibly 2.21) was
   affected and wouldn't have displayed chinese and Arabic characters
   correctly

   Revision 1.20  2000/05/12 03:09:23  sjboddie
   minor modifications to get web library compiling under VC++ 6.0

   Revision 1.19  2000/04/06 19:57:58  cs025
   Correcting a correction - reinstated all lib files due to silly
   CVS confusion.

   Revision 1.17  2000/02/29 21:00:38  sjboddie
   fixed some compiler warnings

   Revision 1.16  1999/11/25 21:53:08  sjboddie
   fixed bug in logout

   Revision 1.15  1999/09/24 02:25:35  rjmcnab
   removed limitation on the number of arguments that a macro can have

   Revision 1.14  1999/09/07 04:57:41  sjboddie
   added gpl notice

   Revision 1.13  1999/08/31 07:59:11  rjmcnab
   Generalised spaces to unicode spaces, added ability to automatically detect
   whether the input file is Unicode or UTF-8 and read the file in the
   appropriate way, and improved the error messages slightly.

   Revision 1.12  1999/07/21 20:46:12  rjmcnab
   fixed small bug

   Revision 1.11  1999/07/21 07:26:30  rjmcnab
   Added more operators to _If_, changed quoting of parameters so that
   ' can be used to quote but it is removed, and made slight optimisation
   so that macros that don't contain other macros aren't expanded.

   Revision 1.10  1999/03/01 20:39:54  sjboddie

   Added eq and ne functionality to _If_

   Revision 1.9  1999/03/01 01:18:09  sjboddie

   Fixed bug in _If_ when value being tested was only one character
   long (e.g. "0" or "1")

   Revision 1.8  1999/02/28 23:15:29  rjmcnab

   Made _If_ recognise 'false' and '0' as being false.

   Revision 1.7  1999/02/08 01:26:11  rjmcnab

   Improved the error reporting.

   Revision 1.6  1999/01/19 08:30:23  rjmcnab

   Added a method to determine whether a macro has been defined.

   Revision 1.5  1999/01/19 01:38:12  rjmcnab

   Made the source more portable.

   Revision 1.4  1999/01/12 01:50:57  rjmcnab

   Standard header.

   Revision 1.3  1999/01/08 02:33:13  rjmcnab

   Added standard header to source files.

 */


#include "display.h"
#include "gsdlunicode.h"
#include "unitool.h"
#include <assert.h>

// include to get NULL
#include <stdlib.h>



/////////////////////////////////////
// misc classes
/////////////////////////////////////


// precedencetype defines a 'precedence value' for each parameter
typedef map<text_t, double, lttext_t> precedencetype;


// macro value structure
struct mvalue
{
  text_t filename;
  text_t value;
};

inline bool operator==(const mvalue &x, const mvalue &y) {
  return (x.filename==y.filename && x.value==y.value);
}

inline bool operator<(const mvalue &x, const mvalue &y) {
  return (x.filename<y.filename ||
      (x.filename==y.filename && x.filename<y.filename));
}



/////////////////////////////////////
// stuff for defaultmacros_t
/////////////////////////////////////
  
typedef map<text_t, mvalue, lttext_t> defparammap;
typedef map<text_t, defparammap, lttext_t> defmacromap;
typedef map<text_t, defmacromap, lttext_t> defpackagemap;

// all methods in defaultmacros_t assume the parameters
// and packages in a standard form and not empty
class defaultmacros_t
{
protected:
  defpackagemap macros;

public:
  typedef defpackagemap::const_iterator const_package_iterator;
  typedef defpackagemap::size_type package_size_type;

  // setmacro returns 0 if there was no error,
  //            -1 if it redefined a macro
  //            -2 if it hid a Global macro
  //            -3 if it redefined a macro and hid a Global macro
  //            -4 if either a package, macroname, or params were not supplied
  int setmacro(const text_t &package,
      const text_t &macroname, 
      const text_t &params, 
      const text_t &filename, 
      const text_t &macrovalue);

  void delmacro(const text_t &package,
          const text_t &macroname, 
          const text_t &params);

  void clear () {macros.erase(macros.begin(), macros.end());}

  // top level stuff
  const_package_iterator package_begin () const {return macros.begin();}
  const_package_iterator package_end () const {return macros.end();}
  bool package_empty () const {return macros.empty();}
  package_size_type package_size() const {return macros.size();}

  // methods to find things
  defmacromap *package_find (const text_t &packagename);
  defparammap *macro_find (const text_t &packagename, 
               const text_t &macroname);
  mvalue *parameter_find (const text_t &packagename, 
              const text_t &macroname,
              const text_t &parametername);
};


// setmacro returns 0 if there was no error,
//            -1 if it redefined a macro
//            -2 if it hid a Global macro
//            -3 if it redefined a macro and hid a Global macro
//            -4 if either a package, macroname, or params were not supplied
int defaultmacros_t::setmacro(const text_t &package,
             const text_t &macroname, 
             const text_t &params, 
             const text_t &filename, 
             const text_t &macrovalue)
{
  paramhashtype paramhash;

  defmacromap *macromapptr;
  defparammap *parammapptr;
  mvalue *mvalueptr;

  int warning = 0;

  // check to see if we would be hiding a Global macro with the 
  // same name. Note: it doesn't matter what parameters are used
  // a Global macro will still be hid by a non-Global one with
  // the same name.
  if ((package != "Global") && 
      (macro_find ("Global", macroname) != NULL))
    {
      warning -= 2; // found the macroname
    }

  // define this macro
  macromapptr = &(macros[package]); // -- package
  parammapptr = &((*macromapptr)[macroname]); // -- macro name
  // -- parameters
  if ((*parammapptr).find(params) != (*parammapptr).end()) {
    warning -= 1; // found the parameters
  }
  mvalueptr = &((*parammapptr)[params]);

  // -- value
  (*mvalueptr).filename = filename;
  (*mvalueptr).value = macrovalue;

  return warning;
}

void defaultmacros_t::delmacro(const text_t &package,
                 const text_t &macroname, 
                 const text_t &params)
{
  // make sure everything was supplied
  if (package.empty() || macroname.empty() || params.empty()) return;

  // find the package and macroname
  defparammap *parammapptr = macro_find(package, macroname);
  if (parammapptr == NULL) return;

  // find the parameters
  defparammap::iterator parammapit = parammapptr->find(params);
  if (parammapit == parammapptr->end()) return;

  // finally delete this element
  parammapptr->erase(parammapit);
}

defmacromap *defaultmacros_t::package_find (const text_t &packagename)
{
  if (packagename.empty()) return NULL;

  defpackagemap::iterator it = macros.find(packagename);
  if (it == macros.end()) return NULL;

  return &((*it).second);
}

defparammap *defaultmacros_t::macro_find (const text_t &packagename, 
                           const text_t &macroname)
{
  defmacromap *macromapptr = package_find(packagename);
  if (macromapptr == NULL || macroname.empty()) return NULL;

  defmacromap::iterator it = (*macromapptr).find(macroname);
  if (it == (*macromapptr).end()) return NULL;

  return &((*it).second);
}

mvalue *defaultmacros_t::parameter_find (const text_t &packagename, 
                     const text_t &macroname,
                     const text_t &parametername)
{
  defparammap *parammapptr = macro_find(packagename, macroname);
  if (parammapptr == NULL || parametername.empty()) return NULL;

  defparammap::iterator it = (*parammapptr).find(parametername);
  if (it == (*parammapptr).end()) return NULL;

  return &((*it).second);
}



/////////////////////////////////////
// stuff for currentmacros_t
/////////////////////////////////////

typedef map<text_t, mvalue, lttext_t> curmacromap;
typedef map<text_t, curmacromap, lttext_t> curpackagemap;

// all methods in currentmacros_t assume the parameters
// and packages in a standard form and not empty
class currentmacros_t 
{
protected:
  curpackagemap macros;

public:
  typedef curpackagemap::const_iterator const_package_iterator;
  typedef curpackagemap::size_type package_size_type;

  // setmacro returns 0 if there was no error,
  //            -1 if it redefined a macro
  //            -4 if either a package, or macroname were not supplied
  int setmacro(const text_t &package,
      const text_t &macroname, 
      const text_t &filename, 
      const text_t &macrovalue);

  void delmacro(const text_t &package,
        const text_t &macroname);

  void clear () {macros.erase(macros.begin(), macros.end());}

  // top level stuff
  const_package_iterator package_begin () const {return macros.begin();}
  const_package_iterator package_end () const {return macros.end();}
  bool package_empty () const {return macros.empty();}
  package_size_type package_size() const {return macros.size();}

  // methods to find things
  curmacromap *package_find (const text_t &packagename);
  mvalue *macro_find (const text_t &packagename, 
              const text_t &macroname);
};


// setmacro returns 0 if there was no error,
//            -1 if it redefined a macro
//            -4 if either a package, or macroname were not supplied
int currentmacros_t::setmacro(const text_t &package,
             const text_t &macroname, 
             const text_t &filename, 
             const text_t &macrovalue)
{
  int warning = 0;

  // make sure everything was supplied
  if (package.empty() || macroname.empty()) return -4;

  // define this macro
  curmacromap *macromapptr = &(macros[package]);
  if ((*macromapptr).find(macroname) != (*macromapptr).end()) 
    {
      warning -= 1; // found the macroname
    }
  mvalue *mvalueptr = &((*macromapptr)[macroname]);

  // -- value
  (*mvalueptr).filename = filename;
  (*mvalueptr).value = macrovalue;

  return warning;
}

void currentmacros_t::delmacro(const text_t &package,
                 const text_t &macroname)
{
  // make sure everything was supplied
  if (package.empty() || macroname.empty()) return;

  // find the package
  curmacromap *macromapptr = package_find(package);
  if (macromapptr == NULL) return;

  // find the macroname
  curmacromap::iterator macromapit = macromapptr->find(macroname);
  if (macromapit == macromapptr->end()) return;

  // finally delete this element
  macromapptr->erase(macromapit);
}

curmacromap *currentmacros_t::package_find (const text_t &packagename)
{
  if (packagename.empty()) return NULL;

  curpackagemap::iterator it = macros.find(packagename);
  if (it == macros.end()) return NULL;

  return &((*it).second);
}

mvalue *currentmacros_t::macro_find (const text_t &packagename, 
                     const text_t &macroname)
{
  curmacromap *macromapptr = package_find(packagename);
  if (macromapptr == NULL || macroname.empty()) return NULL;

  curmacromap::iterator it = (*macromapptr).find(macroname);
  if (it == (*macromapptr).end()) return NULL;

  return &((*it).second);
}




/////////////////////////////////////
// support functions
/////////////////////////////////////

// this calculation of precedence will make one occurance of a high
// precedence item override many occurances of lower precedence items
void calcprecedence (const text_t &precedstr, precedencetype &precedhash) 
{
  text_t param;
  double multiple = 5.0;
  double nextprec = multiple;

  text_t::const_iterator here, end;

  // set precedhash to an empty hash
  precedhash.erase(precedhash.begin(), precedhash.end());

  // iterate through paramstring ignoring space and extracting
  // out key value pairs.
  here = precedstr.begin();
  end = precedstr.end();
  while (here != end) 
    {
      // get the next parameter
      param.clear(); // set param to an empty string
      
      while (here != end) 
    {
      if (*here == ',') 
        {
          // found the end of the parameter
          here++;
          break;
        } 
      else if (*here == ' ') 
        {
          // do nothing (ignore spaces)
        } 
      else 
        {
          // character in parameter
          param.push_back (*here); // copy this character
        }
      
      here++;
    }

      // if a parameter was found insert its value
      if (!param.empty()) 
    {
      precedhash[param] = nextprec;
      nextprec *= multiple;
    }
    }
}


// decides how specific any given parameter is to the page parameters. Returns 
// negative if any options are different or else the sum of the precedence for
// the options which match. 
double getspecificness(const paramhashtype &pageparams, 
               const paramhashtype &theseparams, 
               const precedencetype &precedhash) 
{
  double specificness = 0.0;
  paramhashtype::const_iterator here, pagevalueit;
  precedencetype::const_iterator precedit;
  text_t thesekey, thesevalue, constignore("ignore");
  
  here = theseparams.begin();
  while (here != theseparams.end()) 
    {
      thesekey = (*here).first;
      thesevalue = (*here).second;
      
      if (thesekey == constignore) 
    {
      // do nothing, this is a placeholder
    } 
      else 
    {
      pagevalueit = pageparams.find(thesekey);
      if ((pagevalueit != pageparams.end()) && 
          ((*pagevalueit).second == thesevalue)) 
        {
          // this parameter fits, add its precedence value
          precedit = precedhash.find(thesekey);
          if (precedit == precedhash.end()) 
        specificness += 1.0; // no precedence defined
          else 
          specificness += (*precedit).second;
        }
      else 
        {
          return -1.0; // did not match
        }
    }

      here++;
    }
  
  return specificness;
}


void splitparams (const text_t &paramstring, paramhashtype &paramhash)
{
  text_t key;
  text_t value;

  text_t::const_iterator here, end;

  // set paramhash to an empty hash
  paramhash.erase(paramhash.begin(), paramhash.end());

  // iterate through paramstring ignoring space and extracting
  // out key value pairs.
  here = paramstring.begin();
  end  = paramstring.end();
  while (here != end) 
    {
      // reset key and value
      key.clear();
      value.clear();

      // get key
      while (here != end) 
    {
      if (*here == ',') 
        {
          // have a key with no value
          break;
        } 
      else if (*here == '=') 
        {
          // found the end of the key
          here ++; 
          break;
        } 
      else if (*here == ' ') 
        {
          // do nothing (ignore spaces)
        } 
      else 
        {
          // character in key
          key.push_back (*here); // copy this character
        }
      here ++;
    }

      // get value
      while (here != end) 
    {
      if (*here == '=') 
        {
          // multiple values for one key, ignore previous
          // values
          value.clear();
        } 
      else if (*here == ',') 
        {
          // found the end of the value
          here ++;
          break;
        } 
      else if (*here == ' ') 
        {
          // do nothing (ignore spaces)
        } 
      else 
        {
          // character in value
          value.push_back (*here); // copy this character
        }
      here ++;
    }
    
      // if a key was found insert the key/value pair in 
      // the map 
      if (!key.empty()) 
    {
      paramhash[key] = value;
    }
    }
}


void joinparams (const paramhashtype &paramhash, text_t &paramstring)
{
  //set paramstring to an empty string
  paramstring.clear();

  //iterate through the paramhash
  paramhashtype::const_iterator thepair;
  int firstparam = 1;

  for (thepair=paramhash.begin();thepair!=paramhash.end(); thepair++)
    {
      if (!firstparam) paramstring.push_back(',');
      firstparam = 0;
      
      // insert pairs of "key=value"
      text_t thevalue;
      paramstring.append((*thepair).first);
      paramstring.push_back('=');
      paramstring.append((*thepair).second);
    }
}



/////////////////////////////////////
// parsing support functions
/////////////////////////////////////

inline int my_isalpha (unsigned short c) 
{
  return ((c >= 'A' && c <= 'Z') ||
      (c >= 'a' && c <= 'z'));
} 



inline unsigned short my_ttnextchar (text_t::const_iterator &here, 
                     text_t::const_iterator &end) 
{
  if (here != end) 
    {
      here++;
      if (here != end) return (*here);
    }
  return '\0';
}



/////////////////////////////////////
// methods for display
/////////////////////////////////////

// public methods for display

displayclass::displayclass ()
{
  defaultmacros = new defaultmacros_t;
  defaultfiles = new fileinfomap;
  
  orderparamlist = new paramspeclist;
  currentmacros = new currentmacros_t;

  outc = NULL;

  logout = &cerr;
}


displayclass::~displayclass ()
{
  delete defaultmacros;
  delete defaultfiles;
  
  delete orderparamlist;
  delete currentmacros;
}
  
// setdefaultmacro adds an entry to the list of default macros 
// returns 0 if there was no error,
//        -1 if it redefined a macro
//        -2 if it hid a Global macro
//        -3 if it redefined a macro and hid a Global macro
//        -4 if no macroname was supplied
int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
                  text_t params, const text_t &macrovalue) 
{
  return setdefaultmacro (macroname, package, params, macrovalue, "memory");
}



// as we are using one character lookahead the
// value of line might be off by one.
// the input file must be in the utf-8 or unicode format
// initially for each file isunicode should be set to 0 and 
// bigendian should be set to 1
// 0 will be returned when the end of the file has been found
unsigned short my_uni_get (unistream &fin, int &line, 
               int &isunicode, int &bigendian) {
  unsigned short c = 0;
  
  if (isunicode) {
    // unicode text
    // get the next two characters
    unsigned char c1 = 0, c2 = 0;
    
    if (!fin.eof()) fin.get(c1);
    if (!fin.eof()) fin.get(c2);
    else c1 = 0;

    // if they indicate the order get the next character
    // otherwise just get these characters
    if (c1 == 0xff && c2 == 0xfe) {
      bigendian = 0;
      c = my_uni_get (fin, line, isunicode, bigendian);
    } else if (c1 == 0xfe && c2 == 0xff) {
      bigendian = 1;
      c = my_uni_get (fin, line, isunicode, bigendian);
    } else c = (bigendian) ? (c1*256+c2) : (c2*256+c1);
    
  } else {
    // utf-8 text
    // how many characters we get depends on what we find
    unsigned char c1 = 0, c2 = 0, c3 = 0;
    while (!fin.eof()) {
      fin.get(c1);
      if (c1 == 0xfe || c1 == 0xff) {
    // switch to unicode
    isunicode = 1;
    if (!fin.eof()) fin.get(c2);

    if (c1 == 0xff && c2 == 0xfe) bigendian = 0;
    else bigendian = 1;

    c = my_uni_get (fin, line, isunicode, bigendian);
    break;

      } else if (c1 <= 0x7f) {
    // one byte character
    c = c1;
    break;

      } else if (c1 >= 0xc0 && c1 <= 0xdf) {
    // two byte character
    if (!fin.eof()) fin.get(c2);
    c = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
    break;

      } else if (c1 >= 0xe0 && c1 <= 0xef) {
    // three byte character
    if (!fin.eof()) fin.get(c2);
    if (!fin.eof()) fin.get(c3);
    c = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) + (c3 & 0x3f);
    break;
      }

      // if we get here there was an error in the file, we should
      // be able to recover from it however, maybe the file is in
      // another encoding
    }
  }

  if (c == '\n') line++;
  return c;
}



// loads a default macro file (if it isn't already loaded)
// returns 0 if didn't need to load the file (it was already loaded)
//         1 if was (re)loaded
//        -1 an error occurred while trying to load the file
int displayclass::loaddefaultmacros (text_t thisfilename) {
  // convert the filename to a C string
  char *filenamestr = thisfilename.getcstr();
  outconvertclass text_t2ascii;

  // see if we need to open this file -- this isn't implemented yet
  
  // open the file
  unistream fin (filenamestr);

  if (fin.fail()) return -1; // read failed

  text_t package = "Global";
  int line = 1;
  int isunicode = 0, bigendian = 1;

  // pre-fetch the next character
  unsigned short c = my_uni_get(fin, line, isunicode, bigendian);

  text_t macropackage, macroname, macroparameters, macrovalue;
  int err; // for keeping track of whether an error occurred somewhere

  while (!fin.eof()) {
    // expect: white space, comment, "package", or macroname
    if (is_unicode_space(c)) {
      // found some white-space
      c = my_uni_get(fin, line, isunicode, bigendian);

    } else if (c == '#') {
      // found the start of a comment
      // skip all characters up to the end of the line
      c = my_uni_get(fin, line, isunicode, bigendian); // skip the '#'
      while (!fin.eof ()) {
    if (c == '\n') break;
    c = my_uni_get(fin, line, isunicode, bigendian);
      }

    } else if (c == 'p') {
      // found the start of 'package' (hopefully)
      // get everything up to the next space
      text_t tmp;
      while (!fin.eof() && my_isalpha(c)) {
    tmp.push_back(c);
    c = my_uni_get(fin, line, isunicode, bigendian);
      }
      // see if we have a package name
      if (tmp == "package") {
    // skip all white space
    while (!fin.eof() && is_unicode_space(c)) 
      c = my_uni_get(fin, line, isunicode, bigendian);
    
    // get the package name
    tmp.clear(); // init tmp
    while (!fin.eof() && my_isalpha(c)) {
      tmp.push_back(c);
      c = my_uni_get(fin, line, isunicode, bigendian);
    }
    package = tmp;
    if (package.empty()) package = "Global";

      } else {
    // error
    if (logout != NULL) {
      (*logout) << text_t2ascii << "Expected 'package' on line " << line 
            << " of " << thisfilename << "\n";
    }
      }
      
    } else if (c == '_') {
      // found the start of a macro (hopefully)
      c = my_uni_get(fin, line, isunicode, bigendian); // skip the _

      // init variables
      err = 0;
      macropackage = package;
      macroname.clear(); // init macroname
      macroparameters.clear(); // init macroname
      macrovalue.clear(); // init macroname
      
      // get the macro name
      while ((!fin.eof()) && (!is_unicode_space(c)) && 
         (c != '\\') && (c != '_') &&(c != ':') && 
         (macroname.size() < 80)) {
    macroname.push_back(c);
    c = my_uni_get(fin, line, isunicode, bigendian);
      }
      
      if (c == ':') {
    // we actually had the macro package
    c = my_uni_get(fin, line, isunicode, bigendian); // skip :
    macropackage = macroname;
    macroname.clear ();
    
    // get the macro name (honest!)
    while ((!fin.eof()) && (!is_unicode_space(c)) && 
           (c != '\\') && (c != '_') &&(c != ':') && 
           (macroname.size() < 80)) {
      macroname.push_back(c);
      c = my_uni_get(fin, line, isunicode, bigendian);
    }
      }
      
      if (!err && c == '_') {
    c = my_uni_get(fin, line, isunicode, bigendian); // skip the _
    
    // skip all white space
    while (!fin.eof() && is_unicode_space(c)) c = my_uni_get(fin, line, isunicode, bigendian);
      } else if (!err) err = 1;

      // get the macro parameters (optional)
      if (!err && c == '[') {
    c = my_uni_get(fin, line, isunicode, bigendian); // skip the [
    while ((!fin.eof()) && (c != '\n') && (c != '\\') && (c != ']')) {
      macroparameters.push_back(c);
      c = my_uni_get(fin, line, isunicode, bigendian);
    }
    
    if (c == ']') {
      c = my_uni_get(fin, line, isunicode, bigendian); // skip the ]
      
      // skip all white space
      while (!fin.eof() && is_unicode_space(c)) c = my_uni_get(fin, line, isunicode, bigendian);
    } 
    else if (!err) err = 2;
      }

      // get the macro value
      if (!err && c == '{') {
    c = my_uni_get(fin, line, isunicode, bigendian); // skip the {
    while ((!fin.eof()) && (c != '}')) {
      if (c == '\\') {
        macrovalue.push_back(c); // keep the '\'
        c = my_uni_get(fin, line, isunicode, bigendian); // store the *next* value regardless
        if (!fin.eof()) macrovalue.push_back(c);
        c = my_uni_get(fin, line, isunicode, bigendian);
      }
      macrovalue.push_back(c);
      c = my_uni_get(fin, line, isunicode, bigendian);
    }
    
    if (c == '}') {
      c = my_uni_get(fin, line, isunicode, bigendian); // skip the }
      
      // define the macro
      err = setdefaultmacro (macropackage, macroname, macroparameters,
                 thisfilename, macrovalue);
      if ((err == -1 || err == -3) && logout != NULL) {
        (*logout) << text_t2ascii << "Warning: redefinition of _" << 
          package << ":" << macroname << "_[" << macroparameters <<
          "] on line ";
        (*logout) << line;
        (*logout) << text_t2ascii << " of " << thisfilename << "\n";
        
      } else if (err == -2 && logout != NULL) {
        (*logout) << text_t2ascii << "Warning: _" << 
          package << ":" << macroname << "_[" << macroparameters <<
          "] on line ";
        (*logout) << line;
        (*logout) << text_t2ascii << " of " << 
          thisfilename << " hides a Global macro with the same name\n";

      } else if (err == -4 && logout != NULL) {
        (*logout) << text_t2ascii << "Error: macro name expected on line ";
        (*logout) << line ;
        (*logout) << text_t2ascii << " of " << thisfilename << "\n";
      }       
      
      err = 0; // for the test below
    }
    else if (!err) err = 3;
      }
      else if (!err) err = 4;
      
      if (err) {
    // found an error, skip to the end of the line
    if (logout != NULL) {
      (*logout) << text_t2ascii << "Error: ";
      if (err == 1) (*logout) << text_t2ascii << "'_'";
      else if (err == 2) (*logout) << text_t2ascii << "']'";
      else if (err == 3) (*logout) << text_t2ascii << "'}'";
      else if (err == 4) (*logout) << text_t2ascii << "'{'";
      (*logout) << text_t2ascii << " expected on line ";
      (*logout) << line ;
      (*logout) << text_t2ascii << " of " << thisfilename << "\n";
    }
    while (!fin.eof ()) {
      if (c == '\n') break;
      c = my_uni_get(fin, line, isunicode, bigendian);
    }
      }
      
    } else {
      // found an error, skip to the end of the line
      if (logout != NULL) {
    (*logout) << text_t2ascii << "Error: Unexpected input on line " << line 
          << " of " << thisfilename << "\n";
      }
      while (!fin.eof ()) {
    if (c == '\n') break;
    c = my_uni_get(fin, line, isunicode, bigendian);
      }
      
    }
  }
  
  fin.close ();
  
  // free up memory
  delete filenamestr;
  return 0;
}


// prepares to create a page.
void displayclass::openpage (const text_t &thispageparams, 
            const text_t &thisprecedence)
{
  // init variables
  currentmacros->clear();

  // reload any default macro files which have changed.
  if (checkdefaultmacrofiles() < 0) 
    {
      // print error message
    }

  setpageparams (thispageparams, thisprecedence);
}


// changes the parameters for the current page.
void displayclass::setpageparams (text_t thispageparams, 
                 text_t thisprecedence)
{
  paramhashtype thissplitparams, splittheseparams;
  precedencetype precedhash;
  text_tset::iterator text_tsetit;
  paramspec tmpps;

  precedence = thisprecedence;
  calcprecedence(thisprecedence, precedhash);

  splitparams(thispageparams, thissplitparams);
  joinparams(thissplitparams, thispageparams);
  params = thispageparams;

  // get a list of parameters with their specificness
  orderparamlist->erase(orderparamlist->begin(), orderparamlist->end());
  for (text_tsetit=allparams.begin();
       text_tsetit!=allparams.end();
       text_tsetit++) 
    {
      tmpps.param = (*text_tsetit);
      splitparams(tmpps.param, splittheseparams);
      tmpps.spec = getspecificness(thissplitparams, splittheseparams, precedhash);
      if (tmpps.spec >= 0) 
    {
      orderparamlist->push_back (tmpps);
    }
    }

  // sort the list
  sort(orderparamlist->begin(), orderparamlist->end());

//   paramspeclist::iterator pshere = orderparamlist->begin();
//   paramspeclist::iterator psend = orderparamlist->end();
//   while (pshere != psend) 
//     {
//       cerr << text_t2ascii << "param=" << (*pshere).param;
//       cerr << " spec=" << (int)((*pshere).spec) << "\n";
//       pshere++;
//     }
}


// overrides (or sets) a macro for the current page.
// returns 0 if there was no error,
//        -1 if it redefined a macro
//        -4 if no macroname was supplied
int displayclass::setmacro (const text_t &macroname, 
               text_t package, 
               const text_t &macrovalue)
{
  // make sure a macroname was supplied
  if (macroname.empty()) return -4;

  // make package "Global" if it doesn't point to anything yet
  if (package.empty()) package = "Global";

  // set the macro
  return currentmacros->setmacro(package, macroname, "memory", macrovalue);
}


void displayclass::expandstring (const text_t &inputtext, text_t &outputtext)
{
  expandstring("", inputtext, outputtext);
}


void displayclass::expandstring (text_t package, const text_t &inputtext,
                 text_t &outputtext, int recursiondepth)
{
  text_t macroname, macropackage, macroargs;
  text_t::const_iterator tthere = inputtext.begin();
  text_t::const_iterator ttend = inputtext.end();
  unsigned short c = '\0';

  if (package.empty()) package = "Global";

  outputtext.clear();
      
  // use one-character lookahead
  if (tthere != ttend) c = (*tthere);
  while (tthere != ttend) 
    {
      if (c == '\\')
    {
      // found an escape character
      c = my_ttnextchar (tthere, ttend); // skip the escape character
      if (tthere != ttend) 
        {
          if (c == 'n') outputtext.push_back('\n');
          else if (c == 't') outputtext.push_back('\t');
          else outputtext.push_back(c);
        }
      c = my_ttnextchar (tthere, ttend);
      
    } 
      else if (c == '_') 
    {
      // might have found the start of a macro
      
      // embedded macros (macros within the names
      // of other macros) are no longer supported -- Rodger.
      
      // save the current state (in case we need to back out)
      text_t::const_iterator savedtthere = tthere;
      unsigned short savedc = c;
      
      macroname.clear();
      macropackage = package;
      macroargs.clear();
      
      c = my_ttnextchar (tthere, ttend); // skip the '_'
      
      // get the macroname
      while (tthere != ttend && (!is_unicode_space(c)) && 
         (c != '\\') && (c != '_') &&(c != ':') && 
         (macroname.size() < 80)) 
        {
          macroname.push_back((unsigned char)c);
          c = my_ttnextchar (tthere, ttend);
        }
      
      if (c == ':') 
        {
          // we actually had the macro package
          c = my_ttnextchar (tthere, ttend);
          macropackage = macroname;
          macroname.clear ();
    
          // get the macro name (honest!)
          while ((tthere != ttend) && (!is_unicode_space(c)) && 
             (c != '\\') && (c != '_') &&(c != ':') && 
             (macroname.size() < 80)) 
        {
          macroname.push_back((unsigned char)c);
          c = my_ttnextchar (tthere, ttend);
        }
        }
      
      if ((tthere != ttend) && (c == '_') &&
          (macroname.size() > 0)) 
        {
          // found a macro!!!! (maybe)
    
          c = my_ttnextchar (tthere, ttend); // skip the '_'
    
          // get the parameters (if there are any)
          if ((tthere != ttend) && (c == '(')) 
        {
          c = my_ttnextchar (tthere, ttend); // skip '('
      
          // have to be careful of quotes
          unsigned short quote = '\0';
          while (tthere != ttend) 
            {
              if (c == '\\') 
            {
              // have an escape character, save escape 
              // character and next character as well
              macroargs.push_back (c);
              c = my_ttnextchar (tthere, ttend); 
          
            } 
              else if (quote == '\0') 
            {
              // not in a quote at the moment
              if (c == ')') 
                {
                  // found end of the arguments
                  c = my_ttnextchar (tthere, ttend); // skip ')'
                  break;
        
                } 
              else if (c == '\'' || c == '\"') 
                {
                  // found a quote
                  quote = c;
                }
          
            } 
              else 
            {
              // we are in a quote
              if (c == quote) quote = '\0';
            }
        
              // save this character
              if (tthere != ttend) macroargs.push_back (c);
              c = my_ttnextchar (tthere, ttend);
            }
        }
    
          //
          // we now have the macropackage, macroname, and macroargs
          //
    
          // try to expand out this macro
          text_t expandedmacro;
          if (macro (macroname, macropackage, macroargs, expandedmacro, recursiondepth)) 
        {
          // append the expanded macro
          outputtext += expandedmacro;
        } 
          else
        {
          // wasn't a macro
          tthere = savedtthere;
          c = savedc;
          outputtext.push_back(c); // the '_'
          c = my_ttnextchar (tthere, ttend);
        }
    

        }
      else
        {
          // wasn't a macro
          tthere = savedtthere;
          c = savedc;
          outputtext.push_back(c); // the '_'
          c = my_ttnextchar (tthere, ttend);
        }

    }
      else 
    {
      // nothing of interest
      outputtext.push_back(c); // the '_'
      c = my_ttnextchar (tthere, ttend);
    }
    }
}

ostream *displayclass::setlogout (ostream *thelogout) {

  ostream *oldlogout = logout;
  logout = thelogout;
  return oldlogout;
}

// protected methods for display

// reloads any default macro files which have changed
// returns 0 no errors occurred
//        -1 an error occurred while trying to load one of the files
int displayclass::checkdefaultmacrofiles ()
{
  // this isn't implemented yet
  return 0;
}

// isdefaultmacro sees if there is an entry in the list of
// default macros with the given package and macro name
// returns 0 if no macros in the package or in the global package
//           were found
//         1 if no macros in the package were found but a macro
//           in the global package was found
//         2 if a macro in the given package was found
int displayclass::isdefaultmacro (text_t package, const text_t &macroname) {
  // make sure a macroname was supplied
  if (macroname.empty()) return 0;

  // make package "Global" if it doesn't point to anything yet
  if (package.empty()) package = "Global";

  // check the given package
  if (defaultmacros->macro_find(package, macroname) != NULL) return 2;

  // check the Global package
  if (defaultmacros->macro_find("Global", macroname) != NULL) return 1;

  return 0;
}


// setdefaultmacro adds an entry to the list of default macros 
// returns 0 if there was no error,
//        -1 if it redefined a macro
//        -2 if it hid a Global macro
//        -3 if it redefined a macro and hid a Global macro
//        -4 if no macroname was supplied
int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
                  text_t params, const text_t &filename,
                  const text_t &macrovalue) 
{
  // make sure a macroname was supplied
  if (macroname.empty()) return -4;

  // put the parameters in a standard form
  paramhashtype paramhash;
  if (params.empty()) params = "ignore=yes";
  splitparams (params, paramhash);
  joinparams (paramhash, params);

  // make package "Global" if it doesn't point to anything yet
  if (package.empty()) package = "Global";

  // remember these parameters
  allparams.insert (params);

  // remember this filename (this part isn't finished yet -- Rodger).

  // set the macro
  return defaultmacros->setmacro(package, macroname, params, filename, macrovalue);
}


// evaluates a boolean expression
// returns false if expr equals "" or "0". 
// otherwise returns true *unless* expr is 
// format "XXXX" eq/ne "dddd" in which case
// the two quoted strings are compared
bool displayclass::boolexpr (text_t package, const text_t &expr, int recursiondepth) {

  if (expr.empty()) return false;

  text_t expexpr;
  if (package.empty()) package = "Global";  
  expandstring (package, expr, expexpr, recursiondepth);

  if (expexpr.empty() || expexpr == "0") return false;
  if (expr[0] != '\"') return true;

  // don't use expexpr while separating quoted parts of
  // expression just in case expanded out macros contain
  // quotes
  text_t::const_iterator here = expr.begin();
  text_t::const_iterator end = expr.end();

  int quotecount = 0;
  text_t string1, expstring1;
  text_t string2, expstring2;
  text_t op;
  text_t combineop;
  bool result = false; // an empty string is false
  bool result2 = false;

  while (here != end) {
    // get a comparison
    quotecount = 0;
    string1.clear();
    string2.clear();
    op.clear();
    while (here != end) {
      if (*here == '"') quotecount++;
      else if (quotecount == 1) string1.push_back(*here);
      else if ((quotecount == 2) && !is_unicode_space (*here))
    op.push_back(*here);
      else if (quotecount == 3) string2.push_back(*here);
      else if (quotecount >= 4) break;
      here ++;
    }
    
    expandstring (package, string1, expstring1, recursiondepth);
    expandstring (package, string2, expstring2, recursiondepth);

    // get next result
    result2 = true; // any badly formatted string will return true
    if (op == "eq") result2 = (expstring1 == expstring2);
    else if (op == "ne") result2 = (expstring1 != expstring2);
    else if (op == "gt") result2 = (expstring1 > expstring2);
    else if (op == "ge") result2 = (expstring1 >= expstring2);
    else if (op == "lt") result2 = (expstring1 < expstring2);
    else if (op == "le") result2 = (expstring1 <= expstring2);
    else if (op == "==") result2 = (expstring1.getint() == expstring2.getint());
    else if (op == "!=") result2 = (expstring1.getint() != expstring2.getint());
    else if (op == ">") result2 = (expstring1.getint() > expstring2.getint());
    else if (op == ">=") result2 = (expstring1.getint() >= expstring2.getint());
    else if (op == "<") result2 = (expstring1.getint() < expstring2.getint());
    else if (op == "<=") result2 = (expstring1.getint() <= expstring2.getint());

    // combine the results
    if (combineop == "&&") result = (result && result2);
    else if (combineop == "||") result = (result || result2);
    else result = result2;

    // get next combination operator
    combineop.clear();
    while (here != end && *here != '"') {
      if (!is_unicode_space(*here)) combineop.push_back(*here);
      here++;
    }
  }

  return result;
}


// (recursively) expand out a macro
// returns true  if the macro was found
//         false if the macro was not found
bool displayclass::macro (const text_t &macroname, text_t macropackage, 
             const text_t &macroparam, text_t &outputtext,
             int recursiondepth)
{
  outconvertclass text_t2ascii;
  text_tlist splitmacroparam;
  text_t::const_iterator hereit, endit;
  text_t aparam;
  unsigned short c = '\0', quote = '\0';

  //  cerr << "r: " << recursiondepth << "\n";

  // check for deep recursion
  if (recursiondepth >= MAXRECURSIONDEPTH)
    {
      if (logout != NULL)
    (*logout) << "Warning: deep recursion, limiting macro expansion\n";
      return false;
    }

  // check the macropackage
  if (macropackage.empty()) macropackage = "Global";

  // get the parameters (but don't expand them)
  if (macroparam.size() > 0) {
    // set up for loop
    hereit = macroparam.begin();
    endit = macroparam.end();
    if (hereit != endit) c = (*hereit);

    while (hereit != endit) {
      // get the next parameter
      aparam.clear();
      quote = '\0'; // not-quoted unless proven quoted
      
      // ignore initial whitespace
      while ((hereit!=endit)&&is_unicode_space(c)) c=my_ttnextchar(hereit,endit);
      
      // look for the end of the parameter
      while (hereit != endit) {
    if (c == '\\') {
      // found escape character, also get next character
      aparam.push_back(c);
      c = my_ttnextchar (hereit, endit);
      if (hereit != endit) aparam.push_back(c);
      
    } else if (quote=='\0' && (c=='\'' /*|| c=='"'*/)) {
      // found a quoted section
      quote = c;
      //      aparam.push_back(c);

    } else if (quote!='\0' && c==quote) {
      // found the end of a quote
      quote = '\0';
      //      aparam.push_back(c);

    } else if (quote=='\0' && c==',') {
      // found the end of a parameter
      c = my_ttnextchar (hereit, endit);
      break;

    } else {
      // ordinary character
      aparam.push_back(c);
    }

    c = my_ttnextchar (hereit, endit);
      }

      // add this parameter to the list
      splitmacroparam.push_back(aparam);
    }
  }

  if (macroname == "If") {
    // get the condition, then clause and else clause
    text_tlist::iterator paramcond = splitmacroparam.begin();
    text_tlist::iterator paramend = splitmacroparam.end();
    text_tlist::iterator paramthen = paramend;
    text_tlist::iterator paramelse = paramend;
    
    if (paramcond != paramend) {
      paramthen = paramcond;
      paramthen++;
    }
    if (paramthen != paramend) {
      paramelse = paramthen;
      paramelse++;
    }
    
    // will always output something
    outputtext.clear();
    
    // expand out the first parameter
    if (paramcond != paramend) {
      text_t tmpoutput;
      expandstring (macropackage, *paramcond, tmpoutput, recursiondepth+1);
      lc (tmpoutput);
      
      // test the expanded string
      if (boolexpr (macropackage, *paramcond, recursiondepth+1)) {
    //(tmpoutput.size()) && (tmpoutput != "false") && (tmpoutput != "0")) {
    // true
    if (paramthen != paramend)
      expandstring (macropackage, *paramthen, outputtext, recursiondepth+1);

      } else {
    // false
    if (paramelse != paramend)
      expandstring (macropackage, *paramelse, outputtext, recursiondepth+1);
      }
    }
    
    return true;
  }

  // try and find this macro

  // this list might be replaced by something a little more
  // sophisticated in the future.
  text_tlist packagelist;
  packagelist.push_back (macropackage);
  packagelist.push_back ("Global");

  paramspeclist::iterator paramhere, paramend;
  text_tlist::iterator packagehere = packagelist.begin();
  text_tlist::iterator packageend = packagelist.end();
  mvalue *macrovalue = NULL;
  while ((macrovalue == NULL) && (packagehere != packageend)) 
    {
      // first look in the currentmacros
      macrovalue = currentmacros->macro_find(*packagehere, macroname);
      if (macrovalue == NULL)
    macrovalue = currentmacros->macro_find("Style", macroname);

      // look in the default macros
      if (macrovalue == NULL) 
    {
      // next look in the defaultmacros
      if (defaultmacros->macro_find (*packagehere, macroname) != NULL)
        {
          paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
          while ((macrovalue == NULL) && (paramhere != paramend))
        {
//        cerr << text_t2ascii << "\npackage: " << *packagehere << "\n";
//        cerr << text_t2ascii << "macroname: " << macroname << "\n";
//        cerr << text_t2ascii << "param: " << (*paramhere).param << "\n";
//        cerr << "spec: " << (int)((*paramhere).spec) << "\n";
          macrovalue = defaultmacros->parameter_find(*packagehere, macroname,
                                 (*paramhere).param);
          paramhere++;
        }
        }
    }

      // and again in the package "Style"
      if (macrovalue == NULL) 
    {
      // next look in the defaultmacros
      if (defaultmacros->macro_find ("Style", macroname) != NULL)
        {
          paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
          while ((macrovalue == NULL) && (paramhere != paramend))
        {
          macrovalue = defaultmacros->parameter_find("Style", macroname,
                                 (*paramhere).param);
          paramhere++;
        }
        }
    }
      if (macrovalue == NULL) packagehere++;
    }

  if (macrovalue != NULL)
    {
      // we found a macro

      // replace all the option macros (_1_, _2_, ...) in the value of this macro
      // and decide if we need to expand the value of this macro
      bool needsexpansion = false;
      text_t tempmacrovalue;
      tempmacrovalue.clear();
      hereit = macrovalue->value.begin();
      endit = macrovalue->value.end();
      if (hereit != endit) c = (*hereit);
      while (hereit != endit)
    {
      if (c == '\\')
        {
          // found an escape character
          needsexpansion = true;
          tempmacrovalue.push_back(c);
          c = my_ttnextchar (hereit, endit);
          if (hereit != endit) tempmacrovalue.push_back(c);
          c = my_ttnextchar (hereit, endit);
        }
      else if (c == '_')
        {
          text_t::const_iterator savedhereit = hereit;

          // attempt to get a number
          int argnum = 0;
          unsigned short digc = my_ttnextchar (hereit, endit);
          while (digc >= '0' && digc <= '9') {
        argnum = argnum*10 + digc - '0';
        digc = my_ttnextchar (hereit, endit);
          }
          if (digc == '_' && argnum > 0) {
        // found an option macro, append the appropriate text
        text_tlist::iterator ttlisthere = splitmacroparam.begin();
        text_tlist::iterator ttlistend = splitmacroparam.end();
        while ((argnum > 1) && (ttlisthere != ttlistend)) {
          argnum--;
          ttlisthere++;
        }
        if (ttlisthere != ttlistend) {
          tempmacrovalue.append(*ttlisthere);
          if (findchar((*ttlisthere).begin(), (*ttlisthere).end(), '_') != (*ttlisthere).end())
            needsexpansion = true;
        }

        c = my_ttnextchar (hereit, endit);

          } else {
        // wasn't a option macro
        needsexpansion = true;
        tempmacrovalue.push_back(c);
        hereit = savedhereit;
        c = my_ttnextchar (hereit, endit);
          }
        }
      else
        {
          tempmacrovalue.push_back(c);
          c = my_ttnextchar (hereit, endit);
        }
    }
      
      // recursively replace this macro (if we need to)
      if (needsexpansion) {
    expandstring (*packagehere, tempmacrovalue, outputtext, recursiondepth+1);
      } else {
    outputtext += tempmacrovalue;
      }

      return true;
    }

  if (logout != NULL) {
    (*logout) << text_t2ascii << "Warning: _" << 
      macropackage << ":" << macroname << "_ not found\n";
  }
  
  return false; // didn't find the macro
}


void displayclass::printdefaultmacros ()
{
  defpackagemap::const_iterator packagemapit;
  defmacromap::const_iterator macromapit;
  defparammap::const_iterator parammapit;
  const mvalue *mvalueptr;
  outconvertclass text_t2ascii;

  text_t packagename, macroname, macroparam;
  
  for (packagemapit = defaultmacros->package_begin();
       packagemapit != defaultmacros->package_end();
       packagemapit++) 
    {
      packagename = (*packagemapit).first;
      //     cerr << "***package   : \"" << packagename << "\"\n";
    
      for (macromapit = (*packagemapit).second.begin();
       macromapit != (*packagemapit).second.end();
       macromapit++) 
    {
      macroname = (*macromapit).first;
      //       cerr << "***macroname : \"" << macroname << "\"\n";

      for (parammapit = (*macromapit).second.begin();
           parammapit != (*macromapit).second.end();
           parammapit++) 
        {
          macroparam = (*parammapit).first;
          mvalueptr = &((*parammapit).second);
          cerr << text_t2ascii << "package   : \"" << packagename << "\"\n";
          cerr << text_t2ascii << "macroname : \"" << macroname << "\"\n";
          cerr << text_t2ascii << "parameters: [" << macroparam << "]\n";
          cerr << text_t2ascii << "filename  : \"" << mvalueptr->filename << "\"\n";
          cerr << text_t2ascii << "value     : {" << mvalueptr->value << "}\n\n";
        }
    }
    }
}

void displayclass::printallparams ()
{
  text_tset::iterator text_tsetit;
  outconvertclass text_t2ascii;

  cerr << text_t2ascii << "** allparams\n";
  for (text_tsetit=allparams.begin();
       text_tsetit!=allparams.end();
       text_tsetit++) {
    cerr << text_t2ascii << (*text_tsetit) << "\n";
  }
  cerr << text_t2ascii << "\n";
}



/////////////////////////////////////
// stuff to do tricky output
/////////////////////////////////////

displayclass &operator<< (outconvertclass &theoutc, displayclass &display)
{
  display.setconvertclass (&theoutc);
  return display;
}

displayclass &operator<< (displayclass &display, const text_t &t)
{
  outconvertclass *theoutc = display.getconvertclass();

  if (theoutc == NULL) return display;
  
  text_t output;
  display.expandstring (t, output);
  (*theoutc) << output;

  return display;
}