source: trunk/gsdl/src/recpt/collectoraction.cpp@ 1485

/**********************************************************************
 *
 * collectoraction.cpp -- 
 * Copyright (C) 2000  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

// note that the collectoraction relies on having direct access to a
// collections configuration file. this breaks the separation between
// receptionist and collection server and so is not suitable (at least
// in its current form) for use when collection servers are separate 
// from the receptionist (e.g. when using the CORBA protocol).

#include "collectoraction.h"
#include "OIDtools.h"
#include "fileutil.h"
#include "cfgread.h"
#include "gsdltools.h"

#if !defined (__WIN32__)
#include <sys/utsname.h>
#include <unistd.h>
#endif

collectoraction::collectoraction () {

  recpt = NULL;
  do_mkcol = false;

  cgiarginfo arg_ainfo;
  arg_ainfo.shortname = "a";
  arg_ainfo.longname = "action";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "collector";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "cp";
  arg_ainfo.longname = "collector page";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "intro";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // temporary directory name for this collector
  // session
  arg_ainfo.shortname = "bc1tmp";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1fullname";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1dirname";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1contactemail";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1aboutdesc";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1clone";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1clonecol";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // set when cloning option has changed
  arg_ainfo.shortname = "bc1clonechanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputdir1";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputdir2";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputdir3";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputdir4";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // only set when one of the fields was changed in
  // the "collection info" page
  arg_ainfo.shortname = "bc1infochanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // only set when cfg file is altered from within
  // "configure collection" page
  arg_ainfo.shortname = "bc1cfgchanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1dodelete";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // will be set if we arrived at the "configure collection" page
  // via the "changing an existing collection" page
  arg_ainfo.shortname = "bc1econf";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // will be set if we arrived at the "source data" page
  // via the "changing an existing collection" page
  arg_ainfo.shortname = "bc1esrce";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);
}

collectoraction::~collectoraction () {
  delete gsdlosc;
  delete gsdlhomec;
}

bool collectoraction::init (ostream & /*logout*/) {

  // set up GSDLOS and GSDLHOME environment variables
  text_t gsdlos;
#if defined (__WIN32__)
  gsdlos = "windows";
#else
  utsname *buf = new utsname();
  int i = uname (buf);
  if (i == -1) gsdlos = "linux"; // uname failed
  else gsdlos.setcstr (buf->sysname);
  delete buf;
  lc (gsdlos);
#endif
  
  gsdlos = "GSDLOS=" + gsdlos;
  text_t setgsdlhome = "GSDLHOME=" + gsdlhome;

  // these will be cleaned up in the destructor
  gsdlosc = gsdlos.getcstr();
  gsdlhomec = setgsdlhome.getcstr();
  
  putenv (gsdlosc);
  putenv (gsdlhomec);

  return true;
}

bool collectoraction::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &args, 
                     ostream &logout) {

  text_t &current_page = args["cp"];

  // note that the "bildstatus" and "bildframe1" pages don't actually do anything
  // functional so we don't need to worry about authenticating them (it's the 
  // underlying "bild" page that does the building (and creates the frameset))
  // This helps us overcome a bit of a problem we have with multiple pages trying
  // to read from the key.db database at the same time.
  if (current_page != "intro" && current_page != "bildstatus" && current_page != "bildframe1") {
    // authenticate the user if authentication is available
    args["uan"] = 1;
    args["ug"] = "colbuilder";
  }
  
  if (current_page == "new" || current_page == "existing") {

    // assign (and create) a temporary directory 
    assign_tmpname (args, logout);

    // clean up any old builds left laying about in the tmp directory
    // (note that it's possible this could take some time if there's a huge
    // partially built collection laying about -- may need to think about doing
    // this in the background).
    text_t cleantmp_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "cleantmp.pl");
    char *cleantmp_cmdc = cleantmp_cmd.getcstr();
    system (cleantmp_cmdc);
    delete cleantmp_cmdc;
  }

  if (args["bc1infochanged"] == "1") {
    
    if (args["bc1dirname"].empty()) {
      // we've just come from the "collection information" page for the 
      // first time so we'll need to create the collection with mkcol.pl
      // and set up bc1dirname - we do this part here instead of in do_action
      // because the bc1dirname argument must be set to its new value before
      // the compressedoptions macros are set.
      args["bc1dirname"] = get_directory_name (args["bc1fullname"]);

      text_t createfile = filename_cat (gsdlhome, "tmp", args["bc1tmp"], ".create");
      if (!file_exists (createfile)) {
    // we could do the mkcol.pl here but I guess it's nicer to do it in do_action()
    do_mkcol = true; 
      } else {
    // .create file already exists but bc1dirname wasn't set ... this should only be
    // able to occur when the "reload" (and possibly the "back" and "forward" buttons)
    // have been used to get us here.
    // we'll check that the bc1dirname directory exists (in case of the unlikely 
    // possibility that get_directory_name returned a different value this time
    // than it did originally).
    text_t coldir = filename_cat (get_collectdir(args), args["bc1dirname"]);
    if (!directory_exists (coldir)) {
      message = "reloaderror";
      return true;
    }
      }
    } else {
      // "collection information" has been changed after collection already exists
      // so we'll need to update the cfg file.
      update_cfgfile_partial (args, logout);
    }
  }

  if (args["bc1cfgchanged"] == "1") {
    // configuration file has been changed from the "configure collection"
    // page. we need to update the file on disk and catch bc1 arguments up
    // with changes.
    update_cfgfile_complete (args, logout);
  }
  
  if (args["bc1clonechanged"] == "1") {
    // cloning option has been changed on "source data" page. if it was turned 
    // on we want to create a new collect.cfg file using the bc1clonecol cfg file
    // as a model (we'll save the old file as collect.cfg.org). if cloning was
    // turned off we'll revert to using the collect.cfg.org file (which will need
    // updating in case the bc1 arguments have been altered since cloning was 
    // turned on).
    update_cfgfile_clone (args, logout);

    // if cloning has just been turned on we'll also copy the rest of the files 
    // (excluding collect.cfg which we've already done) from the cloned collections
    // etc directory to the new collection.
    if (args["bc1clone"] == "1") {
      text_t clone_etc = filename_cat(gsdlhome, "collect", args["bc1clonecol"], "etc");
      text_t new_etc = filename_cat(get_collectdir(args), args["bc1dirname"], "etc");
      text_tarray files;

      if (read_dir (clone_etc, files)) {
    text_tarray::const_iterator here = files.begin();
    text_tarray::const_iterator end = files.end();
    while (here != end) {
      if (*here != "collect.cfg" && *here != "collect.cfg.org") {
        file_copy (filename_cat(clone_etc, *here), filename_cat(new_etc, *here));
      }
      here ++;
    }
      } else {
    outconvertclass text_t2ascii;
    logout <<text_t2ascii << "collectoraction::check_cgiargs couldn't read from " 
           << clone_etc << " directory\n";
      }
    }
  }
  
  if (current_page == "bildstatus") {
    // if .final file exists then build has finished
    text_t fbld = filename_cat (gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld.final");
    if (file_exists (fbld)) {
      char *fbldc = fbld.getcstr();
      ifstream fbld_in (fbldc);
      if (fbld_in) {
    char c = fbld_in.get();
    fbld_in.close();
    if (c == '0') current_page = "bilddone";
    else if (c == '4' || c == '5') message = "installfail";
    else current_page = "bildfail";
      } else {
    // assume build failed (we shouldn't get here though ... right?)
    current_page = "bildfail";
      }
      delete fbldc;
    }
  }

  return true;
}

void collectoraction::update_cfgfile_clone (cgiargsclass &args, ostream &logout) {

  text_t tmpdir = filename_cat(gsdlhome, "tmp", args["bc1tmp"]);
  text_t cfgfile = filename_cat(tmpdir, args["bc1dirname"], "etc", "collect.cfg");
  if (!file_exists (cfgfile)) {
    message = "tmpfail";
    return;
  }

  text_t cfgfile_org = filename_cat (tmpdir, "collect.cfg.org");

  if (args["bc1clone"] == "1") {
    // cloning was turned on

    text_t cfgfile_clone = filename_cat(gsdlhome, "collect", args["bc1clonecol"], "etc", "collect.cfg");
    if (file_exists (cfgfile_clone)) {
      // if .org file doesn't exist already create it
      if (!file_exists (cfgfile_org)) {
    if (!file_copy (cfgfile, cfgfile_org)) {
      message = "tmpfail";
      return;
    }
      }
      // copy clone collections cfg file to new collection
      if (!file_copy (cfgfile_clone, cfgfile)) {
    message = "tmpfail";
    return;
      }
      // update the new cfg file
      update_cfgfile_partial (args, logout);
      
    } else {
      // can't clone non-existant or read-protected collection
      message = "clonefail";
    }

  } else {
    // cloning has been turned off having been on at some point. the .org file
    // should exist, if it doesn't we'll bail out and leave the user with the 
    // cloned copy
    if (file_exists (cfgfile_org)) {
      // copy original back again and update it with any recent changes
      if (file_copy (cfgfile_org, cfgfile)) {
    update_cfgfile_partial (args, logout);
      } else {
    message = "tmpfail";
      }
    }
  }
}

// update configuration file on disk to match bc1 arguments
void collectoraction::update_cfgfile_partial (cgiargsclass &args, ostream &logout) {

  text_t cfgfile = filename_cat(get_collectdir(args), args["bc1dirname"], "etc", "collect.cfg");
  char *cfgfilec = cfgfile.getcstr();

  vector<text_tarray> cfgarray;

  // read in cfg file
  ifstream cfg_in (cfgfilec);
  if (cfg_in) {
    text_tarray cfgline;
    while (read_cfg_line(cfg_in, cfgline) >= 0) {
      if (cfgline.size () >= 2) {
    if (cfgline[0] == "creator" || cfgline[0] == "maintainer") {
      cfgline[1] = args["bc1contactemail"];
    } else if (cfgline[0] == "collectionmeta") {
      if (cfgline[1] == "collectionname") {
        cfgline[2] = args["bc1fullname"];
      } else if (cfgline[1] == "collectionextra") {
        cfgline[2] = carriage_replace (args["bc1aboutdesc"], 0);
      }
    }
      }
      cfgarray.push_back (cfgline);
    }
    cfg_in.close();
    
    // now write cfg file back out
#ifdef __WIN32__
    ofstream cfg_out (cfgfilec, ios::binary);
#else
    ofstream cfg_out (cfgfilec);
#endif
    if (cfg_out) {
      // lock the file
      int fd = GSDL_GET_FILEDESC(cfg_out);
      int lock_val = 1;
      GSDL_LOCK_FILE (fd);
      if (lock_val != 0) {
    logout << "Error: Couldn't lock file " << cfgfilec << "\n";
    cfg_out.close();
    message = "tmpfail";
    
      } else {

    vector<text_tarray>::const_iterator this_line = cfgarray.begin();
    vector<text_tarray>::const_iterator end_line = cfgarray.end();
    while (this_line != end_line) {
      write_cfg_line (cfg_out, *this_line);
      this_line ++;
    }
    GSDL_UNLOCK_FILE (fd);
    cfg_out.close();
      }

    } else {
      logout << "collectoraction::update_cfgfile_partial: unable to open "
         << cfgfilec << " for output\n";
      message = "tmpfail";
    }
    
  } else {
    logout << "collectoraction::update_cfgfile_partial: unable to open "
       << cfgfilec << " for input\n";
    message = "tmpfail";
  }
  delete cfgfilec;
}

// replace configuration file on disk with that in the bc1cfgfile argument and 
// catch other bc1 arguments up with those the new cfgfile contains
void collectoraction::update_cfgfile_complete (cgiargsclass &args, ostream &logout) {

  text_t cfgfile = filename_cat(get_collectdir(args), args["bc1dirname"], "etc", "collect.cfg");
  char *cfgfilec = cfgfile.getcstr();
#ifdef __WIN32__
  ofstream cfg_out (cfgfilec, ios::binary);
#else
  ofstream cfg_out (cfgfilec);
#endif
  
  if (cfg_out) {
    // lock the file
    int fd = GSDL_GET_FILEDESC(cfg_out);
    int lock_val = 1;
    GSDL_LOCK_FILE (fd);
    if (lock_val != 0) {
      logout << "Error: Couldn't lock file " << cfgfilec << "\n";
      cfg_out.close();
      message = "tmpfail";
      
    } else {
      
      outconvertclass text_t2ascii;
      cfg_out << text_t2ascii << args["bc1cfgfile"];
      GSDL_UNLOCK_FILE (fd);
      cfg_out.close();
      
      // now that we've written the file we'll read it back again and
      // update our bc1 arguments
      ifstream cfg_in (cfgfilec);
      if (cfg_in) {
    text_tarray cfgline;
    while (read_cfg_line(cfg_in, cfgline) >= 0) {
      if (cfgline.size () >= 2) {
        if (cfgline[0] == "creator") {
          args["bc1contactemail"] = cfgline[1];
        } else if (cfgline[0] == "collectionmeta") {
          if (cfgline[1] == "collectionname") {
        args["bc1fullname"] = cfgline[2];
          } else if (cfgline[1] == "collectionextra") {
        args["bc1aboutdesc"] = carriage_replace (cfgline[2], 1);
          }
        }
      }
    }
    cfg_in.close();
      } else {
    logout << "collectoraction::update_cfgfile_complete: unable to open " 
           << cfgfilec << " for input\n";
    message = "tmpfail";
      }
    }
  } else {
    logout << "collectoraction::update_cfgfile_complete: unable to open " 
       << cfgfilec << " for output\n";
    message = "tmpfail";
  }
  delete cfgfilec;
}

void collectoraction::get_cgihead_info (cgiargsclass &/*args*/, recptprotolistclass * /*protos*/,
                    response_t &response,text_t &response_data, 
                    ostream &/*logout*/) {
  response = content;
  response_data = "text/html";
}

// return html for buttons used in collector bar
// color may be "green", "grey", or "yellow"
// type may be:
//   "info" --> "collection information" button
//   "srce" --> "source data" button
//   "conf" --> "configure collection" button
//   "bild" --> "build collection" button
//   "view" --> "view collection" button
// if enabled is true button will be flashy rollover type and 
// will be hyperlinked

text_t collectoraction::get_button (const text_t &thispage, const text_t &color, 
                    const text_t &type, bool enabled) {

  if ((color != "green" && color != "grey" && color != "yellow") ||
      (type != "info" && type != "srce" && type != "conf" && type != "bild" && type != "view"))
    return "";

  text_t prefix = "gc";
  if (color == "grey") prefix = "nc";
  else if (color == "yellow") prefix = "yc";
  
  text_t httpicon = "httpicon" + prefix + type;
  
  if (enabled) {
    text_t gsmacro = "_gsimage_";
    if (thispage == "info" || thispage == "srce" || thispage == "conf") {
      gsmacro = "_gsjimage_";
    } else if (type == "view") {
      // view button is special case as it needs a target=_top
      gsmacro = "_gstimage_";
    }
    return "<td>" + gsmacro + "(_collector:http" + type + "_,_collector:" + httpicon + 
      "of_,_collector:" + httpicon + "on_," + type + ",_collector:text" + type + "_)</td>\n";
  } else {
    return "<td>_icon" + prefix + type + "of_</td>\n";
  }
}

// set the _fullnamemenu_ macro (and _warnindex_ if we're on the "srce" page)
void collectoraction::set_fullnamemenu (displayclass &disp, cgiargsclass &args, 
                    recptprotolistclass *protos, ostream &logout) {

  if (recpt == NULL) {
    logout << "ERROR (collectoraction::set_fullnamemenu): This action does not contain\n"
       << "       information about any receptionists. The method set_receptionist was\n"
       << "       probably not called from the module which instantiated this action.\n";
    return;
  }

  text_t &current_page = args["cp"];
  text_t currentname = args["bc1dirname"];
  if (current_page == "srce") currentname = args["bc1clonecol"];

  text_tarray dirnames;
  text_tarray fullnames;
  vector<bool> write_protected;
  int selected_index = 0;
  int index = 0;

  recptprotolistclass::iterator rprotolist_here = protos->begin();
  recptprotolistclass::iterator rprotolist_end = protos->end();
  while (rprotolist_here != rprotolist_end) {
    if ((*rprotolist_here).p != NULL) {

      // don't include z39.50 collection
      if ((*rprotolist_here).p->get_protocol_name () == "z3950proto") {
    rprotolist_here ++;
    continue;
      }

      text_tarray collist;
      comerror_t err;
      (*rprotolist_here).p->get_collection_list (collist, err, logout);
      if (err == noError) {
    text_tarray::iterator collist_here = collist.begin();
    text_tarray::iterator collist_end = collist.end();
    FilterResponse_t response;
    text_tset metadata;
    metadata.insert ("collectionname");
    while (collist_here != collist_end) {
      ColInfoResponse_t *cinfo = recpt->get_collectinfo_ptr ((*rprotolist_here).p, *collist_here, logout);
      if (cinfo != NULL) {
        text_t collectionname = *collist_here;
        if (!cinfo->collectionmeta["collectionname"].empty()) {
          // get collection name from the collection cfg file
          collectionname = cinfo->collectionmeta["collectionname"];
        } else if (get_info ("collection", *collist_here, metadata, false, 
                 (*rprotolist_here).p, response, logout)) {
          // get collection name from gdbm file
          collectionname = response.docInfo[0].metadata["collectionname"].values[0];
        }
        dirnames.push_back(*collist_here);
        fullnames.push_back(collectionname);
        // check to see if the collection is writable
        if (collection_protected (*collist_here)) write_protected.push_back(true);
        else write_protected.push_back(false);

        if (*collist_here == currentname) selected_index = index;       
        index ++;
      }
      collist_here ++;
    }
      }
    }
    rprotolist_here ++;
  }

  bool first = true;
  text_t warnindex;
  text_t fullnamemenu = "<select name=\"bc1dirname\">\n";
  if (current_page == "srce") fullnamemenu = "<select name=\"bc1clonecol\" onChange=\"menuchange();\">\n";
  for (int i = 0; i < index; i ++) {
    // don't want write protected collections in list on "change existing
    // collection" page
    if (write_protected[i] && current_page == "existing") continue;
    fullnamemenu += "<option value=\"" + dirnames[i] + "\"";
    if (i == selected_index) fullnamemenu += " selected";
    fullnamemenu.push_back ('>');
    fullnamemenu += fullnames[i];
    if (write_protected[i]) fullnamemenu += " <b>(write protected)</b>";
    fullnamemenu.push_back ('\n');
    
    // add to Warnindex if collection uses any dubious plugins 
    // (if creating clone collection list)
    if (current_page == "srce") {
      if (!first) warnindex.push_back(',');
      if (uses_weird_plugin (dirnames[i])) {
    warnindex += text_t (1);
      } else {
    warnindex += text_t (0);
      }
    }
    first = false;
  }
  fullnamemenu += "</select>\n";

  if (!first) {
    disp.setmacro ("fullnamemenu", "collector", fullnamemenu);
    if (current_page == "srce")
      disp.setmacro ("warnindex", "collector", warnindex);
  }
}

// set the _cfgfile_ macro
void collectoraction::set_cfgfile (displayclass &disp, cgiargsclass &args, ostream &logout) {

  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // read in collect.cfg      
  text_t cfgfile = filename_cat(get_collectdir(args), collection, "etc", "collect.cfg");  
  char *cfgfilec = cfgfile.getcstr();

#ifdef GSDL_USE_IOS_H
  ifstream cfg_ifs (cfgfilec, ios::in | ios::nocreate);
#else
  ifstream cfg_ifs (cfgfilec, ios::in);
#endif
  
  if (cfg_ifs) {  
    // read in collect.cfg      
    text_t cfgtext;
    char c;
    cfg_ifs.get(c);
    while (!cfg_ifs.eof ()) {
      if (c=='\\') cfgtext.push_back('\\');
      cfgtext.push_back(c);
      cfg_ifs.get(c);
    }
    cfg_ifs.close();
    
    // define it as a macro
    disp.setmacro("cfgfile","collector",cfgtext);

  } else {
    logout << "collectoraction::set_cfgfile: couldn't open configuration file (" 
       << cfgfilec << ") for reading\n";
    message = "tmpfail";
  }    
  delete cfgfilec;
}

// set the _statusline_ macro
void collectoraction::set_statusline (displayclass &disp, cgiargsclass &args, ostream & /*logout*/) {

  // the build command creates .bld.download, .bld.import, and .bld.build files (in that
  // order) and deletes them (also in that order) when each stage is complete. the .bld
  // file is the concatenation of all these files.
  text_t bld_file = filename_cat (gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld");
  text_t statusline;

  if (file_exists (bld_file + ".download")) {
    statusline = "Downloading files ...<br>\n";
    statusline += file_tail (bld_file + ".download", 1);
  } else if (file_exists (bld_file + ".import")) {
    statusline = "Importing collection ...<br>\n";
    statusline += file_tail (bld_file + ".import", 1);
  } else if (file_exists (bld_file + ".build")) {
    statusline = "Building collection ...<br>\n";
    statusline += file_tail (bld_file + ".build", 1);
  } else {
    statusline += "creating collection ...<br>\n";
    statusline += file_tail (bld_file, 1);
  }

  disp.setmacro ("statusline", "collector", statusline);

}

void collectoraction::define_internal_macros (displayclass &disp, cgiargsclass &args, 
                          recptprotolistclass *protos, ostream &logout) {

  // define_internal_macros sets the following macros:
  // _collectorbar_
  // _pagescriptextra_
  // _fullnamemenu_ -- if displaying the "source data" page or the "changing existing 
  //                   collection" page
  // _cfgfile_ -- if displaying the "configure collection" page
  // _statusline_ -- if displaying the bildstatus page
  // _header_ -- may be set for pages that require it
  // _faillog_ - set to last 6 lines of .bld file if build failed

  text_t &collector_page = args["cp"];
  int esrce = args["bc1esrce"].getint();
  int econf = args["bc1econf"].getint();

  // set _pagescriptextra_ macro to _cpagescriptextra_
  disp.setmacro ("pagescriptextra", "collector", "_" + collector_page + "scriptextra_");

  if (collector_page == "bildstatus" || collector_page == "bilddone" || 
      collector_page == "bildfail" || collector_page == "bildframe1") {
    disp.setmacro ("header", "collector", "_" + collector_page + "header_");
  }

  // set the collectorbar macro
  text_t collectorbar = "<table border=0 cellspacing=4 cellpadding=0><tr>\n";

  if (collector_page == "new") {
    collectorbar += get_button (collector_page, "green", "info", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);

  } else if (collector_page == "info") {
    collectorbar += get_button (collector_page, "yellow", "info", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "srce", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td align=center>_icongreyuparrow_</td><td colspan=8></td>\n";

  } else if (collector_page == "srce") {
    if (esrce == 1) {
      // if we came from the "change an existing collection" page previous button(s)
      // are disabled
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "conf", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td colspan=2></td><td align=center>_icongreyuparrow_</td><td colspan=6></td>\n";

  } else if (collector_page == "conf") {
    // disable appropriate buttons if we came from "change an existing collection"
    // page
    if (esrce == 1 || econf == 1) {
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    if (econf == 1) {
      collectorbar += get_button (collector_page, "grey", "srce", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "srce", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td colspan=4></td><td align=center>_icongreyuparrow_</td><td colspan=4></td>\n";

  } else if (collector_page == "bilddone") {
    // all previous buttons grey after build was completed
    collectorbar += get_button (collector_page, "grey", "info", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "view", true);
    collectorbar += "</tr><tr><td colspan=6></td><td align=center>_icongreyuparrow_</td><td colspan=2></td>\n";

  } else if (collector_page == "bildcancel") {
    // disable appropriate buttons if we came from "change an existing collection"
    // page
    if (esrce == 1 || econf == 1) {
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    if (econf == 1) {
      collectorbar += get_button (collector_page, "grey", "srce", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "srce", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "conf", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
  }

  if (collector_page == "bildfail") {
    text_t bldlog = filename_cat(gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld");
    text_t rawlog = file_tail (bldlog, 6);
    // we'll shove in some <br> tags where \n's occur
    text_t faillog;
    text_t::const_iterator here = rawlog.begin();
    text_t::const_iterator end = rawlog.end();
    while (here != end) {
      if (*here == '\n') faillog += "<br>";
      faillog.push_back (*here);
      here ++;
    }
    disp.setmacro ("faillog", "collector", faillog);
  }

  collectorbar += "</tr></table>\n";
  disp.setmacro ("collectorbar", "collector", collectorbar);

  if (collector_page == "srce" || collector_page == "existing") 
    set_fullnamemenu (disp, args, protos, logout);
  if (collector_page == "conf") 
    set_cfgfile (disp, args, logout);
  if (collector_page == "bildstatus")
    set_statusline (disp, args, logout);
}

bool collectoraction::do_action (cgiargsclass &args, recptprotolistclass * /*protos*/, 
                 browsermapclass * /*browsers*/, displayclass &disp, 
                 outconvertclass &outconvert, ostream &textout, 
                 ostream &logout) {

  text_t &collector_page = args["cp"];
  text_t &collection = args["bc1dirname"];

  if (collector_page == "bild") {
    // do the work (download, import, build)
    gsdl_build (args, logout);

    if (message.empty()) {
      // bild page is a frameset so we don't want headers and stuff
      textout << outconvert << disp << ("_collector:bildcontent_\n");
    }
  }

  if (do_mkcol == true) {
    // execute mkcol.pl (do_mkcol is set from within check_cgiargs)
    gsdl_mkcol (args, logout);
    do_mkcol = false; // reset for fast-cgi
  }

  if (args["bc1dodelete"] == "1") {
    // delete bcidirname collection
    if (collection_protected (collection)) {
      message = "delinvalid";
    
    } else {
    
      text_t delete_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "delcol.pl");
      delete_cmd += " -f " + collection;
      char *delete_cmdc = delete_cmd.getcstr();
      int rv = system (delete_cmdc);
      delete delete_cmdc;
      if (rv != 0) {
    // deletion failed -- permissions?
    message = "delpermission";
      } else {
    message = "delsuccess";
      }
    }
  }
  
  if (collector_page == "bildcancel") {
    // cancel the build
    gsdl_cancel_build (args, logout);
  }

  if (message.empty()) {
    if (collector_page != "bild") {
      // output page ("bild" page was already output above)
      textout << outconvert << disp << ("_collector:header_\n")
          << ("_collector:" + collector_page + "content_\n")
          << ("_collector:footer_\n");
    }
  } else {
    // message was set somewhere (probably an error), output message page
    textout << outconvert << disp << ("_collector:header_\n")
        << ("_collector:" + message + "content_\n")
        << ("_collector:footer_\n");
    message.clear();
  }
  return true;
}

// if sw = 0 replace all carriage returns in intext with the string "\n"
// else replace all occurances of "\n" with a carriage return
text_t collectoraction::carriage_replace (const text_t &intext, int sw) {
   
  text_t outtext;
  text_t::const_iterator here = intext.begin();
  text_t::const_iterator end = intext.end();
  while (here != end) {
    if (sw == 0) {
      if (*here == '\n') {
    if ((here+1) != end && *(here+1) == '\r') here ++;
    outtext += "\\n";
      } else if (*here == '\r') {
    if ((here+1) != end && *(here+1) == '\n') here ++;
    outtext += "\\n";
      } else {
    outtext.push_back (*here);
      }
    } else if (*here == '\\' && (here+1) != end && *(here+1) == 'n') {
      outtext.push_back ('\n');
      here ++;
    } else {
      outtext.push_back (*here);
    }
    here ++;
  }
  return outtext;
}

// create a short directory name from fullname
text_t collectoraction::get_directory_name (const text_t &fullname) {

  text_t shortname;
  if (fullname.empty()) {
    shortname = "coll";

  } else {
    
    // first make all lowercase and remove any dodgy characters
    // (i.e. anything not [a-z]
    text_t::const_iterator here = fullname.begin();
    text_t::const_iterator end = fullname.end();
    while (here != end) {
      if ((*here >= 'A' && *here <= 'Z') || (*here >= 'a' && *here <= 'z') ||
      (*here == ' ')) {
    if (*here >= 'A' && *here <= 'Z') shortname.push_back (*here+32);
    else if (*here == ' ') {
      while ((*(here+1)) == ' ') here ++;
      shortname.push_back (*here);
    } else shortname.push_back (*here);
      }
      here ++;
    }

    text_tarray words;
    splitchar (shortname.begin(), shortname.end(), ' ', words);
    int num_words = words.size();

    if (num_words == 0) {
      shortname = "coll";

    } else {

      shortname.clear();
      int use_words = (num_words <= 6) ? num_words : 6;
      int substr_len = 6 / use_words;
      
      for (int i = 0; i < use_words; i++) {
    if (words[i].size() < substr_len) shortname += words[i];
    else shortname += substr (words[i].begin(), words[i].begin()+substr_len);
      }
    }
  }

  // check to see if shortname is unique
  text_t fulldirname = filename_cat (gsdlhome, "collect", shortname);
  if (directory_exists (fulldirname)) {
    int version = 0;
    text_t newname;
    do {
      version ++;
      newname = shortname;
      newname.push_back ('v');
      newname.appendint (version);
      fulldirname = filename_cat (gsdlhome, "collect", newname);
    } while (directory_exists (fulldirname));

    shortname = newname;
  }
  
  return shortname;
}

// tests if collection is write protected (currently just checks if
// collect.cfg file is writable
bool collectoraction::collection_protected (const text_t &collection) {
  text_t cfgfile = filename_cat(gsdlhome, "collect", collection, "etc", "collect.cfg");
  if (file_writable(cfgfile)) return false;
  return true;
}

// assigns a temporary directory name for this collector session
// and creates temporary directory
void collectoraction::assign_tmpname (cgiargsclass &args, ostream &logout) {

  int i = 0;
  text_t tmpname = "tbuild";
  while (directory_exists (filename_cat (gsdlhome, "tmp", tmpname + text_t(i)))) {
    i++;
  }
  tmpname.appendint (i);

  text_t fulltmpdir = filename_cat (gsdlhome, "tmp", tmpname);
  if (!mk_dir (fulltmpdir)) {
    outconvertclass text_t2ascii;
    logout << text_t2ascii << "collectoraction::assign_tmpname unable to create directory ("
       << fulltmpdir << ")\n";
  }

  args["bc1tmp"] = tmpname;
}

void collectoraction::gsdl_mkcol (cgiargsclass &args, ostream &logout) {

  text_t tmpdir = filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  if (!directory_exists (tmpdir)) {
    message = "tmpfail";
    return;
  }

  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // check for a .create file - if it exists then we've already created the collection
  text_t createfile = filename_cat (tmpdir, ".create");
  if (file_exists (createfile)) {
    return;
  } else {
    // create the .create file (this file is just a place holder to let any future
    // pages know that the collection already exists).
    char *createfilec = createfile.getcstr();
    ofstream cfile_out (createfilec);
    delete createfilec;
    if (cfile_out) {
      cfile_out << "collection created\n";
      cfile_out.close();
    } else {
      message = "tmpfail";
      return;
    }
  }

  // set up options
  text_t options = "-creator \"" + args["bc1contactemail"] + "\"";
  options += " -title \"" + args["bc1fullname"] + "\"";
  options += " -about \"" + carriage_replace (args["bc1aboutdesc"], 0) + "\"";
  options += " -collectdir \"" + tmpdir + "\" ";
  text_t optionfile = filename_cat (tmpdir, "mkcol.opt");
  char *optionfilet = optionfile.getcstr();
  ofstream ofile_out (optionfilet);
  delete optionfilet;
  if (!ofile_out) {
    message = "tmpfail";
    return;
  }
  outconvertclass text_t2ascii;
  ofile_out << text_t2ascii << options << "\n";
  ofile_out.close();
  
  // set up the mkcol.pl command
  text_t mkcol_cmd = "perl ";
  mkcol_cmd += filename_cat (gsdlhome, "bin", "script", "mkcol.pl");
  mkcol_cmd += " -optionfile \"" + optionfile + "\" " + collection;
  char *mkcol_cmdc = mkcol_cmd.getcstr();
  
  // just an ordinary system call running in the foreground. we're assuming (hoping??)
  // that mkcol.pl will run through fast enough that nothing more elaborate is required.
  system (mkcol_cmdc);
  delete mkcol_cmdc;
  
  // make sure it went ok
  text_t cfgfile = filename_cat (tmpdir, collection, "etc", "collect.cfg");
  if (!file_writable (cfgfile)) {
    message = "mkcolfail";
  }
}

void collectoraction::gsdl_build (cgiargsclass &args, ostream &logout) {

  text_t tmpdir = filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  if (!directory_exists (tmpdir)) {
    message = "tmpfail";
    return;
  }
  
  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // check for a .build file - if it exists then we've already built
  // the collection (or are in the process of building it)
  text_t buildfile = filename_cat (tmpdir, ".build");
  if (file_exists (buildfile)) {
    return;
  } else {
    // create the .build file (this file is just a place holder to let any future
    // pages know that we've already been here)
    char *buildfilec = buildfile.getcstr();
    ofstream bfile_out (buildfilec);
    delete buildfilec;
    if (bfile_out) {
      bfile_out << "collection building\n";
      bfile_out.close();
    } else {
      message = "tmpfail";
      return;
    }
  }

  // set up build options
  text_t options = "-remove_import -out \"";
  options += filename_cat (tmpdir, collection + ".bld");
  options += "\" -collectdir \"" + get_collectdir (args) + "\"";

  if (args["bc1esrce"] == 1) {
    // we're adding data to an existing collection
    options += "\" -save_archives -append";
  }

  if (!args["bc1inputdir1"].empty())
    options += " -download \"" + args["bc1inputdir1"] + "\"";
  if (!args["bc1inputdir2"].empty())
    options += " -download \"" + args["bc1inputdir2"] + "\"";
  if (!args["bc1inputdir3"].empty())
    options += " -download \"" + args["bc1inputdir3"] + "\"";
  if (!args["bc1inputdir4"].empty())
    options += " -download \"" + args["bc1inputdir4"] + "\"";
  text_t optionfile = filename_cat (tmpdir, "build.opt");
  char *optionfilet = optionfile.getcstr();
  ofstream ofile_out (optionfilet);
  delete optionfilet;
  if (!ofile_out) {
    message = "tmpfail";
    return;
  }
  outconvertclass text_t2ascii;
  ofile_out << text_t2ascii << options << "\n";
  ofile_out.close();

  // set up the build command
  text_t build_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "build");
  build_cmd += " -optionfile \"" + optionfile + "\" " + collection;
#if !defined (__WIN32__)
  // run in background on unix systems
  //  build_cmd += " &";
#endif
  char *build_cmdc = build_cmd.getcstr();
  //#if defined (__WIN32__)
  gsdl_system (build_cmdc, logout);
  //#else
  //  system (build_cmdc);
  //#endif
  delete build_cmdc;
}

void collectoraction::gsdl_cancel_build (cgiargsclass &args, ostream &logout) {
  // I really wanted to do what this perl script does from within the library
  // c++ code. I ran into some problems though (like how do you write a portable
  // "rm -r" in c++?). One day I'll spend some time sorting it out ... maybe.
  text_t cancel_cmd = "perl " + filename_cat (gsdlhome, "bin", "script", "cancel_build.pl");
  cancel_cmd += " -collectdir \"" + filename_cat (gsdlhome, "tmp", args["bc1tmp"]) + 
    "\" " + args["bc1dirname"];
  char *cancel_cmdc = cancel_cmd.getcstr();
  system (cancel_cmdc);
  delete (cancel_cmdc);
}

text_t collectoraction::get_collectdir (cgiargsclass &args) {

  if ((args["bc1econf"] == 1) || (args["bc1esrce"] == 1)) {
    // we're adding to a collection in place
    return filename_cat(gsdlhome, "collect");  

  } else {
    return filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  }
}

// checks to see if any of the plugins in pluginset occur in 
// collections configuration file
bool collectoraction::uses_weird_plugin (const text_t &collection) {

  text_tset pluginset;
  pluginset.insert ("HBPlug");

  text_t cfgfile_content;
  text_t cfgfile_name = filename_cat (gsdlhome, "collect", collection, "etc", "collect.cfg");
  text_t pluginstr, pluginname;

  if (read_file (cfgfile_name, cfgfile_content)) {
    text_t::const_iterator here = cfgfile_content.begin();
    text_t::const_iterator end = cfgfile_content.end();
    while (here != end) {
      here = findchar (here, end, 'p');
      if (here == end) break;
      if ((here+6 < end) && (substr (here, here+6) == "plugin")) {
    getdelimitstr (here+6, end, '\n', pluginstr);
    text_t::const_iterator hp = pluginstr.begin();
    text_t::const_iterator ep = pluginstr.end();
    bool found = false;
    // remove any leading whitespace, trailing options etc.
    while (hp != ep) {
      if (*hp == '\t' || *hp == ' ' || *hp == '\n') {
        if (found) break;
      } else {
        pluginname.push_back (*hp);
        found = true;
      }
      hp ++;
    }
    text_tset::const_iterator it = pluginset.find (pluginname);
    if (it != pluginset.end()) return true; // found matching plugin
    pluginname.clear();
      }
      here ++;
    }
  }
  return false;
}