source: trunk/gsdl/src/recpt/collectoraction.cpp@ 5017

/**********************************************************************
 *
 * collectoraction.cpp -- 
 * Copyright (C) 2000  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

// note that the collectoraction relies on having direct access to a
// collections configuration file. this breaks the separation between
// receptionist and collection server and so is not suitable (at least
// in its current form) for use when collection servers are separate 
// from the receptionist (e.g. when using the CORBA protocol).

// following line required to get fstream.filedesc() on darwin (Mac OS X)
#define _STREAM_COMPAT  1
// required for utsname on solaris???
#define _XOPEN_SOURCE 1
#define _XOPEN_SOURCE_EXTENDED 1

#include "collectoraction.h"
#include "OIDtools.h"
#include "fileutil.h"
#include "cfgread.h"
#include "gsdltools.h"
#include "gsdltimes.h"
#include "nullproto.h"
#include "argdb.h"
#include "cgiutils.h"
#include <stdio.h>
#include <fcntl.h>

#if !defined (__WIN32__)
#include <sys/utsname.h>
#include <unistd.h>
#endif

collectoraction::collectoraction () {

  recpt = NULL;
  disabled = true;
  do_mkcol = false;
  badsources = false;
  failedsources.erase(failedsources.begin(), failedsources.end());
  gsdlosc = NULL;
  gsdlhomec = NULL;
  pathc = NULL;

  cgiarginfo arg_ainfo;
  arg_ainfo.shortname = "a";
  arg_ainfo.longname = "action";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "collector";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "p";
  arg_ainfo.longname = "page";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "intro";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // temporary directory name for this collector
  // session
  arg_ainfo.shortname = "bc1tmp";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1fullname";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1dirname";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1contactemail";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1aboutdesc";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1clone";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1clonecol";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  // set when cloning option has changed
  arg_ainfo.shortname = "bc1clonechanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; 
  argsinfo.addarginfo (NULL, arg_ainfo);

  // only set when one of the fields was changed in
  // the "collection info" page
  arg_ainfo.shortname = "bc1infochanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // only set when cfg file is altered from within
  // "configure collection" page
  arg_ainfo.shortname = "bc1cfgchanged";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "cfgfile";
  arg_ainfo.longname = "configuration file contents";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1dodelete";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // will be set if we arrived at the "configure collection" page
  // via the "changing an existing collection" page
  arg_ainfo.shortname = "bc1econf";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  // will be set if we arrived at the "source data" page
  // via the "changing an existing collection" page
  arg_ainfo.shortname = "bc1esrce";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::must;
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputnum";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "3";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1input";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  arg_ainfo.shortname = "bc1inputtype";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = true;
  arg_ainfo.multiplevalue = true;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot; // saved on disk
  argsinfo.addarginfo (NULL, arg_ainfo);

  // will be set when we've just come from the "source data" page
  arg_ainfo.shortname = "bc1fromsrce";
  arg_ainfo.longname = "collector specific";
  arg_ainfo.multiplechar = false;
  arg_ainfo.multiplevalue = false;
  arg_ainfo.defaultstatus = cgiarginfo::weak;
  arg_ainfo.argdefault = "0";
  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
  argsinfo.addarginfo (NULL, arg_ainfo);
}

collectoraction::~collectoraction () {
  if (gsdlosc != NULL) delete gsdlosc;
  if (gsdlhomec != NULL) delete gsdlhomec;
  if (pathc != NULL) delete pathc;
}


void collectoraction::configure (const text_t &key, const text_tarray &cfgline) {
  if ((key == "collector") && (cfgline.size() == 1) && 
      (cfgline[0] == "true" || cfgline[0] == "on" || cfgline[0] == "enabled")) {
    disabled = false;
  } else {
    // call the parent class to deal with the things which
    // are not dealt with here
    action::configure (key, cfgline);
  }
}


bool collectoraction::init (ostream & /*logout*/) {

  // set up GSDLOS, GSDLHOME and PATH environment variables
  text_t gsdlos, path;
  unsigned int path_separator = ':';
#if defined (__WIN32__)
  gsdlos = "windows";
  path_separator = ';';

  path = filename_cat (gsdlhome, "bin", "windows", "perl", "bin;");

#else
  struct utsname *buf = new struct utsname();
  int i = uname (buf);
  if (i == -1) gsdlos = "linux"; // uname failed
  else gsdlos.setcstr (buf->sysname);
  delete buf;
  lc (gsdlos);
#endif

  pathc = getenv ("PATH");
  path += filename_cat (gsdlhome, "bin", gsdlos);
  path.push_back (path_separator);
  path += filename_cat (gsdlhome, "bin", "script");
  if (pathc != NULL) {
    path.push_back (path_separator);
    path += pathc;
  }
  path = "PATH=" + path;

  gsdlos = "GSDLOS=" + gsdlos;
  text_t setgsdlhome = "GSDLHOME=" + gsdlhome;

  // these will be cleaned up in the destructor
  gsdlosc = gsdlos.getcstr();
  gsdlhomec = setgsdlhome.getcstr();
  pathc = path.getcstr();
  
  putenv (gsdlosc);
  putenv (gsdlhomec);
  putenv (pathc);

  return true;
}

bool collectoraction::check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
                     recptprotolistclass * /*protos*/, ostream &logout) {

  text_t &current_page = args["p"];

  // note that the "bildstatus" and "bildframe1" pages don't actually do anything
  // functional so we don't need to worry about authenticating them (it's the 
  // underlying "bild" page that does the building (and creates the frameset))
  // This helps us overcome a bit of a problem we have with multiple pages trying
  // to read from the key.db database at the same time.
  if (current_page != "intro" && current_page != "bildstatus" && current_page != "bildframe1") {
    // authenticate the user if authentication is available
    args["uan"] = 1;
    args["ug"] = "colbuilder";
  }
  
  if (current_page == "new" || current_page == "existing") {

    // assign (and create) a temporary directory 
    assign_tmpname (args, logout);

    // clean up any old builds left laying about in the tmp directory
    // (note that it's possible this could take some time if there's a huge
    // partially built collection laying about so we'll make it an asynchronous
    // system call)
    gsdl_system ("perl -S cleantmp.pl", false, logout);
  }

  if (current_page != "intro" && current_page != "bildstatus" && 
      current_page != "bildframe1" && current_page != "new") {
    // update arguments that were saved to the harddrive
    text_tmap saved_args;
    saved_args["bc1fullname"] = "";
    saved_args["bc1contactemail"] = "";
    saved_args["bc1aboutdesc"] = "";
    saved_args["bc1clone"] = "";
    saved_args["bc1clonecol"] = "";
    saved_args["bc1inputnum"] = "";
    saved_args["bc1input"] = "";
    saved_args["bc1inputtype"] = "";

    // update the argdb database with any arguments that were set
    // by previous page
    text_tmap::iterator here = saved_args.begin();
    text_tmap::iterator end = saved_args.end();
    while (here != end) {
      if (args.lookupcgiarg((*here).first).source != cgiarg_t::default_arg) {
    (*here).second = args[(*here).first];
      }
      here++;
    }

    text_t argfile = filename_cat(gsdlhome, "tmp", args["bc1tmp"], "argdb.db");
    argdb *args_on_disk = new argdb(argfile);
    if (!args_on_disk->update_args(saved_args)) {
      // error
      logout << "collectoraction: argdb::update_args failed (" << argfile << ")\n";
    }

    // update args from argdb
    saved_args.erase(saved_args.begin(), saved_args.end());
    if (!args_on_disk->get_args(saved_args)) {
      // error
      logout << "collectoraction: argdb::get_args failed (" << argfile << ")\n";
    }
    delete args_on_disk;
    here = saved_args.begin();
    end = saved_args.end();
    while (here != end) {
      if (!(*here).second.empty()) {
    args[(*here).first] = (*here).second;
      }
      here ++;
    }
  }

  if (args["bc1infochanged"] == "1") {
    
    if (args["bc1dirname"].empty()) {
      // we've just come from the "collection information" page for the 
      // first time so we'll need to create the collection with mkcol.pl
      // and set up bc1dirname - we do this part here instead of in do_action
      // because the bc1dirname argument must be set to its new value before
      // the compressedoptions macros are set.
      args["bc1dirname"] = get_directory_name (args["bc1fullname"]);

      text_t createfile = filename_cat (gsdlhome, "tmp", args["bc1tmp"], ".create");
      if (!file_exists (createfile)) {
    // we could do the mkcol.pl here but I guess it's nicer to do it in do_action()
    do_mkcol = true; 
      } else {
    // .create file already exists but bc1dirname wasn't set ... this should only be
    // able to occur when the "reload" (and possibly the "back" and "forward" buttons)
    // have been used to get us here.
    // we'll check that the bc1dirname directory exists (in case of the unlikely 
    // possibility that get_directory_name returned a different value this time
    // than it did originally).
    text_t coldir = filename_cat (get_collectdir(args), args["bc1dirname"]);
    if (!directory_exists (coldir)) {
      message = "reloaderror";
      return true;
    }
      }
    } else {
      // "collection information" has been changed after collection already exists
      // so we'll need to update the cfg file.
      update_cfgfile_partial (args, false, logout);
    }
  }

  if (args["bc1cfgchanged"] == "1") {
    // configuration file has been changed from the "configure collection"
    // page. we need to update the file on disk and catch bc1 arguments up
    // with changes.
    update_cfgfile_complete (args, logout);
  }
  
  if (args["bc1clonechanged"] == "1") {
    // cloning option has been changed on "source data" page. if it was turned 
    // on we want to create a new collect.cfg file using the bc1clonecol cfg file
    // as a model (we'll save the old file as collect.cfg.org). if cloning was
    // turned off we'll revert to using the collect.cfg.org file (which will need
    // updating in case the bc1 arguments have been altered since cloning was 
    // turned on).
    update_cfgfile_clone (args, logout);

    // if cloning has just been turned on we'll also copy the rest of the files 
    // (excluding collect.cfg which we've already done) from the cloned collections
    // etc directory to the new collection.
    if (args["bc1clone"] == "1") {
      text_t clone_etc = filename_cat(gsdlhome, "collect", args["bc1clonecol"], "etc");
      text_t new_etc = filename_cat(get_collectdir(args), args["bc1dirname"], "etc");
      text_tarray files;

      if (read_dir (clone_etc, files)) {
    text_tarray::const_iterator here = files.begin();
    text_tarray::const_iterator end = files.end();
    while (here != end) {
      if (*here != "collect.cfg" && *here != "collect.cfg.org") {
        file_copy (filename_cat(clone_etc, *here), filename_cat(new_etc, *here));
      }
      here ++;
    }
      } else {
    outconvertclass text_t2ascii;
    logout <<text_t2ascii << "collectoraction::check_cgiargs couldn't read from " 
           << clone_etc << " directory\n";
      }
    }
  }
  
  if (current_page == "bildstatus" || current_page == "bildcancel") {
    // if .final file exists then build has finished
    text_t fbld = filename_cat (gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld.final");
    if (file_exists (fbld)) {
      char *fbldc = fbld.getcstr();
      ifstream fbld_in (fbldc);
      if (fbld_in) {
    failcode = fbld_in.get();
    fbld_in.close();
    if (failcode == '0') {
      // success - we need to create and configure a collection server for the
      // newly built collection (for fastcgi and local library where
      // initialization isn't going to be redone when the user clicks the
      // "view your new collection" button
      create_colserver (args["bc1dirname"], logout);
      current_page = "bilddone";
    }
    else current_page = "bildfail";
      } else {
    // assume build failed (we shouldn't get here though ... right?)
    current_page = "bildfail";
      }
      delete fbldc;
    }
  }

  if (args["bc1fromsrce"] == "1") {

    // we've just come from the "source data" page so we need to check that
    // input sources are valid
    if (!check_sources(args, logout)) {
      args["p"] = "srce";
    }
  }

  return true;
}

void collectoraction::update_cfgfile_clone (cgiargsclass &args, ostream &logout) {

  text_t tmpdir = filename_cat(gsdlhome, "tmp", args["bc1tmp"]);
  text_t cfgfile = filename_cat(tmpdir, args["bc1dirname"], "etc", "collect.cfg");
  if (!file_exists (cfgfile)) {
    message = "tmpfail";
    return;
  }

  text_t cfgfile_org = filename_cat (tmpdir, "collect.cfg.org");

  if (args["bc1clone"] == "1") {
    // cloning was turned on

    text_t cfgfile_clone = filename_cat(gsdlhome, "collect", args["bc1clonecol"], "etc", "collect.cfg");
    if (file_exists (cfgfile_clone)) {
      // if .org file doesn't exist already create it
      if (!file_exists (cfgfile_org)) {
    if (!file_copy (cfgfile, cfgfile_org)) {
      message = "tmpfail";
      return;
    }
      }
      // copy clone collections cfg file to new collection
      if (!file_copy (cfgfile_clone, cfgfile)) {
    message = "tmpfail";
    return;
      }
      // update the new cfg file
      update_cfgfile_partial (args, true, logout);
      
    } else {
      // can't clone non-existant or read-protected collection
      message = "clonefail";
    }

  } else {
    // cloning has been turned off having been on at some point. the .org file
    // should exist, if it doesn't we'll bail out and leave the user with the 
    // cloned copy
    if (file_exists (cfgfile_org)) {
      // copy original back again and update it with any recent changes
      if (file_copy (cfgfile_org, cfgfile)) {
    update_cfgfile_partial (args, false, logout);
      } else {
    message = "tmpfail";
      }
    }
  }
}

// update configuration file on disk to match bc1 arguments
// there's a special case if the clone option is true as certain parts of a
// config file should not be cloned (e.g. the iconcollection stuff)
void collectoraction::update_cfgfile_partial (cgiargsclass &args, bool clone, ostream &logout) {

  text_t cfgfile = filename_cat(get_collectdir(args), args["bc1dirname"], "etc", "collect.cfg");
  char *cfgfilec = cfgfile.getcstr();

#if defined (__WIN32__)
  // make sure collect.cfg isn't read-only
  _chmod (cfgfilec, _S_IREAD | _S_IWRITE);
#endif

  vector<text_tarray> cfgarray;

  // read in cfg file
  ifstream cfg_in (cfgfilec);
  if (cfg_in) {
    text_tarray cfgline;
    while (read_cfg_line(cfg_in, cfgline) >= 0) {
      if (cfgline.size () >= 2) {
    if (cfgline[0] == "creator" || cfgline[0] == "maintainer") {
      cfgline[1] = args["bc1contactemail"];
    } else if (cfgline[0] == "collectionmeta") {
      if (cfgline[1] == "collectionname") {
        cfgline[2] = args["bc1fullname"];
      } else if (cfgline[1] == "collectionextra") {
        cfgline[2] = carriage_replace (args["bc1aboutdesc"], 0);
      } else if (clone && (cfgline[1] == "iconcollection" ||
                   cfgline[1] == "iconcollectionsmall")) {
        cfgline[2] = "";
      }
    }
      }
      cfgarray.push_back (cfgline);
    }
    cfg_in.close();
    
    // now write cfg file back out
    int fd=open(cfgfilec, O_WRONLY | O_CREAT | O_TRUNC
#if defined(__WIN32__)
        | O_BINARY
#endif
        );

    if (fd != -1) {
      // lock the file
      int lock_val = 1;
      GSDL_LOCK_FILE (fd);
      if (lock_val != 0) {
    logout << "Error: Couldn't lock file " << cfgfilec << "\n";
    close(fd);
    message = "tmpfail";
    
      } else {

    vector<text_tarray>::const_iterator this_line = cfgarray.begin();
    vector<text_tarray>::const_iterator end_line = cfgarray.end();
    while (this_line != end_line) {
      write_cfg_line (fd, *this_line);
      this_line ++;
    }
    GSDL_UNLOCK_FILE (fd);
    close(fd);
      }

    } else {
      logout << "collectoraction::update_cfgfile_partial: unable to open "
         << cfgfilec << " for output\n";
      message = "tmpfail";
    }
    
  } else {
    logout << "collectoraction::update_cfgfile_partial: unable to open "
       << cfgfilec << " for input\n";
    message = "tmpfail";
  }

  delete cfgfilec;
}

// replace configuration file on disk with that in the cfgfile argument and 
// catch other bc1 arguments up with those the new cfgfile contains
void collectoraction::update_cfgfile_complete (cgiargsclass &args, ostream &logout) {

  text_t cfgfile = filename_cat(get_collectdir(args), args["bc1dirname"], "etc", "collect.cfg");
  char *cfgfilec = cfgfile.getcstr();

#ifdef __WIN32__
  // make sure collect.cfg isn't read-only
  _chmod (cfgfilec, _S_IREAD | _S_IWRITE);
#endif

  int fd=open(cfgfilec, O_WRONLY | O_CREAT | O_TRUNC
#if defined(__WIN32__)
          | O_BINARY
#endif
          );

  if (fd) {
    // lock the file
    int lock_val = 1;
    GSDL_LOCK_FILE (fd);
    if (lock_val != 0) {
      logout << "Error: Couldn't lock file " << cfgfilec << "\n";
      close(fd);
      message = "tmpfail";
      
    } else {
      
      outconvertclass text_t2ascii;
      text_t2ascii.setinput(&args["cfgfile"]);
      size_t buffersize=args["cfgfile"].size();
      char *buffer=new char[buffersize];
      buffer[0]='\n'; // just in case something goes wrong...
      size_t num_chars;
      convertclass::status_t status;
      text_t2ascii.convert(buffer, buffersize, num_chars, status);
      // ignore status - assume it is "finished" as buffer is big enough
      write(fd, buffer, num_chars);
      GSDL_UNLOCK_FILE (fd);
      close(fd);
      delete buffer;
      
      // now that we've written the file we'll read it back again and
      // update our bc1 arguments
      ifstream cfg_in (cfgfilec);
      if (cfg_in) {
    text_tarray cfgline;
    while (read_cfg_line(cfg_in, cfgline) >= 0) {
      if (cfgline.size () >= 2) {
        if (cfgline[0] == "creator") {
          args["bc1contactemail"] = cfgline[1];
        } else if (cfgline[0] == "collectionmeta") {
          if (cfgline[1] == "collectionname") {
        args["bc1fullname"] = cfgline[2];
          } else if (cfgline[1] == "collectionextra") {
        args["bc1aboutdesc"] = carriage_replace (cfgline[2], 1);
          }
        }
      }
    }
    cfg_in.close();
      } else {
    logout << "collectoraction::update_cfgfile_complete: unable to open " 
           << cfgfilec << " for input\n";
    message = "tmpfail";
      }
    }
  } else {
    logout << "collectoraction::update_cfgfile_complete: unable to open " 
       << cfgfilec << " for output\n";
    message = "tmpfail";
  }

  delete cfgfilec;
}

void collectoraction::get_cgihead_info (cgiargsclass &/*args*/, recptprotolistclass * /*protos*/,
                    response_t &response,text_t &response_data, 
                    ostream &/*logout*/) {
  response = content;
  response_data = "text/html";
}

// return html for buttons used in collector bar
// color may be "green", "grey", or "yellow"
// type may be:
//   "info" --> "collection information" button
//   "srce" --> "source data" button
//   "conf" --> "configure collection" button
//   "bild" --> "build collection" button
//   "view" --> "view collection" button
// if enabled is true button will be flashy rollover type and 
// will be hyperlinked

text_t collectoraction::get_button (const text_t &thispage, const text_t &color, 
                    const text_t &type, bool enabled) {

  if ((color != "green" && color != "grey" && color != "yellow") ||
      (type != "info" && type != "srce" && type != "conf" && type != "bild" && type != "view"))
    return "";

  text_t prefix = "gc";
  if (color == "grey") prefix = "nc";
  else if (color == "yellow") prefix = "yc";
  
  text_t httpicon = "httpicon" + prefix + type;
  
  if (enabled) {
    text_t gsmacro = "_gsimage_";
    if (thispage == "info" || thispage == "srce" || thispage == "conf" || 
    thispage == "bildcancel" || thispage == "bildfail") {
      gsmacro = "_gsjimage_";
    } else if (type == "view") {
      // view button is special case as it needs a target=_top
      gsmacro = "_gstimage_";
    }
    return "<td>" + gsmacro + "(_collector:http" + type + "_,_collector:" + httpicon + 
      "of_,_collector:" + httpicon + "on_," + type + ",_collector:text" + type + "_)</td>\n";
  } else {
    return "<td>_icon" + prefix + type + "of_</td>\n";
  }
}

// set the _fullnamemenu_ macro (and _warnindex_ and _selectedindex_ if
// we're on the "srce" page)
void collectoraction::set_fullnamemenu (displayclass &disp, cgiargsclass &args, 
                    recptprotolistclass *protos, ostream &logout) {

  if (recpt == NULL) {
    logout << "ERROR (collectoraction::set_fullnamemenu): This action does not contain\n"
       << "       information about any receptionists. The method set_receptionist was\n"
       << "       probably not called from the module which instantiated this action.\n";
    return;
  }

  text_t &current_page = args["p"];
  text_t currentname = args["bc1dirname"];
  if (current_page == "srce") currentname = args["bc1clonecol"];

  text_tarray dirnames;
  text_tarray fullnames;
  vector<bool> write_protected;
  bool is_selected = false;
  int selected_index = 0;
  int index = 0;

  recptprotolistclass::iterator rprotolist_here = protos->begin();
  recptprotolistclass::iterator rprotolist_end = protos->end();
  while (rprotolist_here != rprotolist_end) {
    if ((*rprotolist_here).p != NULL) {

      // don't include z39.50 collections
      comerror_t err = noError;
      if ((*rprotolist_here).p->get_protocol_name (err) == "z3950proto") {
    rprotolist_here ++;
    continue;
      }

      text_tarray collist;
      (*rprotolist_here).p->get_collection_list (collist, err, logout);
      if (err == noError) {
    text_tarray::iterator collist_here = collist.begin();
    text_tarray::iterator collist_end = collist.end();
    FilterResponse_t response;
    text_tset metadata;
    metadata.insert ("collectionname");
    while (collist_here != collist_end) {
      ColInfoResponse_t *cinfo = recpt->get_collectinfo_ptr ((*rprotolist_here).p, *collist_here, logout);
      if (cinfo != NULL) {
        text_t collectionname = *collist_here;
        if (!cinfo->collectionmeta["collectionname"].empty()) {
          // get collection name from the collection cfg file
          collectionname = cinfo->collectionmeta["collectionname"];
        } else if (get_info ("collection", *collist_here, metadata, false, 
                 (*rprotolist_here).p, response, logout)) {
          // get collection name from gdbm file
          collectionname = response.docInfo[0].metadata["collectionname"].values[0];
        }
        dirnames.push_back(*collist_here);
        fullnames.push_back(collectionname);
        // check to see if the collection is writable
        if (collection_protected (*collist_here)) write_protected.push_back(true);
        else write_protected.push_back(false);

        if (*collist_here == currentname) {
          is_selected = true;
          selected_index = index;       
        }
        index ++;
      }
      collist_here ++;
    }
      }
    }
    rprotolist_here ++;
  }

  bool first = true;
  text_t warnindex;
  text_t fullnamemenu = "<select name=\"bc1dirname\">\n";
  if (current_page == "srce") {
    fullnamemenu = "<select name=\"bc1clonecol\" onChange=\"menuchange();\">\n";
    fullnamemenu += "<option value=defaultstructure";
    if (!is_selected) fullnamemenu += " selected>";
    else fullnamemenu.push_back('>');
    fullnamemenu += "_collector:textdefaultstructure_\n";
  }
  for (int i = 0; i < index; i ++) {
    // don't want write protected collections in list on "change existing
    // collection" page
    if (write_protected[i] && current_page == "existing") continue;
    fullnamemenu += "<option value=\"" + dirnames[i] + "\"";
    if ((i == 0 && !is_selected && current_page != "srce") || 
    (is_selected && i == selected_index)) {
      fullnamemenu += " selected";
      selected_index++;
      is_selected = false;
    }
    fullnamemenu.push_back ('>');
    fullnamemenu += fullnames[i];
    fullnamemenu.push_back ('\n');
    
    // add to Warnindex if collection uses any dubious plugins 
    // (if creating clone collection list)
    if (current_page == "srce") {
      if (first) warnindex += "0,";
      else warnindex.push_back(',');
      if (uses_weird_plugin (dirnames[i])) {
    warnindex += text_t (1);
      } else {
    warnindex += text_t (0);
      }
    }
    first = false;
  }
  fullnamemenu += "</select>\n";

  disp.setmacro ("fullnamemenu", "collector", fullnamemenu);
  if (current_page == "srce") {
    disp.setmacro ("warnindex", "collector", warnindex);
    disp.setmacro ("selectedindex", "collector", text_t(selected_index));
  }
}

// set _sourcelist_ and _badsources_ macros
void collectoraction::set_inputsourceboxes (displayclass &disp, cgiargsclass &args, 
                        ostream &logout) {
  
  if (badsources) disp.setmacro ("badsources", "collector", "1");

  text_t sourcelist = get_source_box(args["bc1input"], args["bc1inputnum"].getint(),
                     args["bc1inputtype"]);

  disp.setmacro("sourcelist", "collector", sourcelist);

  // reset badsources and failedsources variables
  badsources = false;
  failedsources.erase(failedsources.begin(), failedsources.end());
}

text_t collectoraction::get_source_box (text_t inputarglist, int numboxes,
                    text_t inputtypelist) {

  text_tarray inputvalues;
  splitchar (inputarglist.begin(), inputarglist.end(), ',', inputvalues);
  // remove any empty values from the end of the array
  if (inputvalues.size()) {
    text_tarray::iterator l = inputvalues.end() - 1;
    text_tarray::iterator b = inputvalues.begin();
    while ((*l).empty() && l >= b) {
      l--;
    }
    inputvalues.erase(l+1, inputvalues.end());
  }

  text_tarray inputtypes;
  splitchar (inputtypelist.begin(), inputtypelist.end(), ',', inputtypes);

  int numvalues = inputvalues.size();
  int numtypes = inputtypes.size();

  text_t last = "file://";
  text_t rv;
  for (int i = 0; i < numboxes; i++) {
    rv += "<nobr><select name=\"bc1inputtype\">\n";
    rv += "<option value=\"file://\"";
    if ((i < numtypes && inputtypes[i] == "file://") ||
    (numboxes == 3 && i == 0 && numvalues == 0) ||
    (i >= 3 && i >= numvalues && last == "file://")) {
      rv += " selected";
      last = "file://";
    }
    rv += ">file://\n";
    rv += "<option value=\"http://\"";
    if ((i < numtypes && inputtypes[i] == "http://") ||
    (numboxes == 3 && i == 1 && numvalues == 0) ||
    (i >= 3 && i >= numvalues && last == "http://")) {
      rv += " selected";
      last = "http://";
    }
    rv += ">http://\n";
    rv += "<option value=\"ftp://\"";
    if ((i < numtypes && inputtypes[i] == "ftp://") ||
    (numboxes == 3 && i == 2 && numvalues == 0) ||
    (i >= 3 && i >= numvalues && last == "ftp://")) {
      rv += " selected";
      last = "ftp://";
    }
    rv += ">ftp://\n";
    rv += "</select>\n";
    rv += "<input type=text name=\"bc1input\" value=\"";
    if (i < numvalues) {
      rv += dm_safe(decode_commas(inputvalues[i]));
    }
    rv += "\" size=50>";
    if (badsources) {
      if ((i < numvalues) && (!inputvalues[i].empty())) {
    if (failedsources[decode_commas(inputvalues[i])] == "1") {
      rv += "_iconcross_";
    } else {
      rv += "_icontick_";
    }
      } else {
    rv += "_iconblank_";
      }
    }
    if (i+1 == numboxes) {
      if (!badsources) rv += "_iconblank_";
      rv += "_imagemore_</nobr><br>";
    } else {
      rv += "</nobr><br>\n";
    }
  }
  
  return rv;
}

// set the _cfgfile_ macro
void collectoraction::set_cfgfile (displayclass &disp, cgiargsclass &args, ostream &logout) {

  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // read in collect.cfg      
  text_t cfgfile = filename_cat(get_collectdir(args), collection, "etc", "collect.cfg");  
  char *cfgfilec = cfgfile.getcstr();

#ifdef GSDL_USE_IOS_H
  ifstream cfg_ifs (cfgfilec, ios::in | ios::nocreate);
#else
  ifstream cfg_ifs (cfgfilec, ios::in);
#endif
  
  if (cfg_ifs) {  
    // read in collect.cfg      
    text_t cfgtext;
    char c;
    cfg_ifs.get(c);
    while (!cfg_ifs.eof ()) {
      cfgtext.push_back(c);
      cfg_ifs.get(c);
    }
    cfg_ifs.close();
    
    // define it as a macro
    disp.setmacro("cfgfile", "collector", dm_safe(cfgtext));

  } else {
    logout << "collectoraction::set_cfgfile: couldn't open configuration file (" 
       << cfgfilec << ") for reading\n";
    message = "tmpfail";
  }    
  delete cfgfilec;
}

// set the _statusline_ macro
void collectoraction::set_statusline (displayclass &disp, cgiargsclass &args, ostream & /*logout*/) {

  // the build command creates .bld.download, .bld.import, and .bld.build files (in that
  // order) and deletes them (also in that order) when each stage is complete. the .bld
  // file is the concatenation of all these files.
  text_t bld_file = filename_cat (gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld");
  text_t statusline;

  if (file_exists (bld_file + ".download")) {
    statusline = "_collector:textdownloadingfiles_<br>\n";
    statusline += dm_safe(file_tail(bld_file + ".download", 1, 0));
  } else if (file_exists (bld_file + ".import")) {
    statusline = "_collector:textimportingcollection_<br>\n";
    statusline += dm_safe(file_tail(bld_file + ".import", 1, 0));
  } else if (file_exists (bld_file + ".build")) {
    statusline = "_collector:textbuildingcollection_<br>\n";
    statusline += dm_safe(file_tail(bld_file + ".build", 1, 0));
  } else {
    statusline += "_collector:textcreatingcollection_<br>\n";
    statusline += dm_safe(file_tail(bld_file, 1, 0));
  }

  disp.setmacro ("statusline", "collector", statusline);

}

void collectoraction::define_internal_macros (displayclass &disp, cgiargsclass &args, 
                          recptprotolistclass *protos, ostream &logout) {

  // define_internal_macros sets the following macros:
  // _collectorbar_
  // _pagescriptextra_
  // _fullnamemenu_ -- if displaying the "source data" page or the "changing existing 
  //                   collection" page
  // _cfgfile_ -- if displaying the "configure collection" page
  // _statusline_ -- if displaying the bildstatus page
  // _header_ -- may be set for pages that require it
  // _textfailmsg_ -- set to different messages depending on failcode returned
  //                  by build script (if build fails)
  // _faillog_ - set to last 6 lines of .bld file if build failed
  // _gsdlhome_ - the gsdlhome path (dm_safe)
  // _sourcelist_ -- "input source" text boxes
  // _badsources_ -- will be set to "1" if we've come from the 
  //                 "source data" page and there's a problem 
  //                 with the input sources

  text_t &collector_page = args["p"];
  int esrce = args["bc1esrce"].getint();
  int econf = args["bc1econf"].getint();

  // set _pagescriptextra_ macro to _cpagescriptextra_
  disp.setmacro ("pagescriptextra", "collector", "_" + collector_page + "scriptextra_");

  if (collector_page == "bildstatus" || collector_page == "bilddone" || 
      collector_page == "bildfail" || collector_page == "bildframe1") {
    disp.setmacro ("header", "collector", "_" + collector_page + "header_");
  }

  // set the collectorbar macro
  text_t collectorbar = "<table border=0 cellspacing=4 cellpadding=0><tr>\n";

  if (collector_page == "new") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "info", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);

  } else if (collector_page == "info") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "info", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "srce", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td></td><td align=center>_icongreyuparrow_</td><td colspan=8></td>\n";

  } else if (collector_page == "srce") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    if (esrce == 1) {
      // if we came from the "change an existing collection" page previous button(s)
      // are disabled
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "conf", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td colspan=3></td><td align=center>_icongreyuparrow_</td><td colspan=6></td>\n";

  } else if (collector_page == "conf") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    // disable appropriate buttons if we came from "change an existing collection"
    // page
    if (esrce == 1 || econf == 1) {
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    if (econf == 1) {
      collectorbar += get_button (collector_page, "grey", "srce", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "srce", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
    collectorbar += "</tr><tr><td colspan=5></td><td align=center>_icongreyuparrow_</td><td colspan=4></td>\n";

  } else if (collector_page == "bilddone") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    // all previous buttons grey after build was completed
    collectorbar += get_button (collector_page, "grey", "info", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "srce", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "conf", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "bild", false);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "green", "view", true);
    collectorbar += "</tr><tr><td colspan=7></td><td align=center>_icongreyuparrow_</td><td colspan=2></td>\n";

  } else if (collector_page == "bildcancel" || collector_page == "bildfail") {
    collectorbar += "<td>_icongreyarrow_</td>\n";
    // disable appropriate buttons if we came from "change an existing collection"
    // page
    if (esrce == 1 || econf == 1) {
      collectorbar += get_button (collector_page, "grey", "info", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "info", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    if (econf == 1) {
      collectorbar += get_button (collector_page, "grey", "srce", false);
    } else {
      collectorbar += get_button (collector_page, "yellow", "srce", true);
    }
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "conf", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "yellow", "bild", true);
    collectorbar += "<td>_icongreyarrow_</td>\n";
    collectorbar += get_button (collector_page, "grey", "view", false);
  }

  collectorbar += "</tr></table>\n";
  disp.setmacro ("collectorbar", "collector", collectorbar);

  if (collector_page == "bildfail") {

    text_t textfailmsg = "_textfailmsg";
    textfailmsg.push_back(failcode);
    textfailmsg.push_back('_');
    disp.setmacro("textfailmsg", "collector", textfailmsg);

    text_t bldlog = filename_cat(gsdlhome, "tmp", args["bc1tmp"], args["bc1dirname"] + ".bld");
    text_t rawlog = file_tail (bldlog, 6, 0);
    // we'll shove in some <br> tags where \n's occur
    text_t faillog;
    text_t::const_iterator here = rawlog.begin();
    text_t::const_iterator end = rawlog.end();
    while (here != end) {
      if (*here == '\n') faillog += "<br>";
      faillog.push_back (*here);
      here ++;
    }
    disp.setmacro ("faillog", "collector", dm_safe(faillog));
  }

  if (collector_page == "srce" || collector_page == "existing") 
    set_fullnamemenu (disp, args, protos, logout);
  if (collector_page == "conf") 
    set_cfgfile (disp, args, logout);
  if (collector_page == "bildstatus")
    set_statusline (disp, args, logout);
  if (collector_page == "srce") {
    set_inputsourceboxes (disp, args, logout);
  }

  disp.setmacro ("gsdlhome", "collector", dm_safe(gsdlhome));
}

bool collectoraction::do_action (cgiargsclass &args, recptprotolistclass * /*protos*/, 
                 browsermapclass * /*browsers*/, displayclass &disp, 
                 outconvertclass &outconvert, ostream &textout, 
                 ostream &logout) {

  // make sure the collector is enabled
  if (disabled) {
    textout << outconvert 
        << "<html>\n"
        << "<head>\n"
        << "<title>Collector disabled</title>\n"
        << "</head>\n"
        << "<body bgcolor=\"#ffffff\" text=\"#000000\" link=\"#006666\" "
        << "alink=\"#cc9900\" vlink=\"#666633\">\n"
        << "<h2>Facility disabled</h2>\n" 
        << "Sorry, the Collector end-user collection building facility is currently disabled\n"
        << "\n</body>\n"
        << "</html>\n";
    return true;
  }

  text_t &collector_page = args["p"];
  text_t &collection = args["bc1dirname"];

  // make sure we have perl (we won't bother with this check for the 
  // building status pages to avoid slowing things down unneccessarily)
  if (collector_page != "bildstatus" && collector_page != "bildframe1" && !perl_ok(logout)) {
    textout << outconvert 
        << "<html>\n"
        << "<head>\n"
        << "<title>Perl not found</title>\n"
        << "</head>\n"
        << "<body bgcolor=\"#ffffff\" text=\"#000000\" link=\"#006666\" "
        << "alink=\"#cc9900\" vlink=\"#666633\">\n"
        << "<h2>Perl not found</h2>\n" 
        << "Greenstone could not detect perl on this system. It is therefore not\n"
        << "possible to build a Greenstone collection, either from the Collector or the \n"
        << "command-line tools, or to use the Collector for any other task.\n"
        << "<p>Please refer to the Greenstone Installer's Guide for details on\n"
        << "installing perl on your system.\n"
        << "\n</body>\n"
        << "</html>\n";
    return true;

  }

  if (collector_page == "bild") {
    // do the work (download, import, build)
    gsdl_build (args, logout);

    if (message.empty()) {
      // bild page is a frameset so we don't want headers and stuff
      textout << outconvert << disp << ("_collector:bildcontent_\n");
    }
  }

  if (do_mkcol == true) {
    // execute mkcol.pl (do_mkcol is set from within check_cgiargs)
    gsdl_mkcol (args, logout);
    do_mkcol = false; // reset for fast-cgi
  }

  if (args["bc1dodelete"] == "1") {
    // delete bcidirname collection
    if (collection_protected (collection)) {
      message = "delinvalid";
    
    } else {

      const recptconf &rcinfo = recpt->get_configinfo ();
      bool emailuserevents = rcinfo.EmailUserEvents;

      // get collection maintainer email from collect.cfg before we
      // delete it
      text_t colmaintainer;
      text_t cfgfile = filename_cat(gsdlhome, "collect", collection, "etc", "collect.cfg");
      char *cfgfilec = cfgfile.getcstr();
      ifstream cfg_in (cfgfilec);
      delete cfgfilec;
      if (cfg_in) {
    text_tarray cfgline;
    while (read_cfg_line(cfg_in, cfgline) >= 0) {
      if (cfgline.size () == 2 && cfgline[0] == "maintainer") {
        colmaintainer = cfgline[1];
        break;
      }
    }
    cfg_in.close();
      }
      if (colmaintainer.empty()) {
    logout << outconvert 
           << "collectoraction::do_action WARNING: Collection being deleted (" 
           << collection << ") has no maintainer address. EmailUserEvents "
           << "disabled\n";
    emailuserevents = false;
      }
      
      // first we need to free up the collection's collection server
      // we must do this for the local library (and I guess when using
      // fastcgi too) as you can't delete the gdbm file while it's
      // being kept open by the collection server
      remove_colservr (collection, logout);

      text_t delete_cmd = "perl -S delcol.pl -f " + collection;
      int rv = gsdl_system (delete_cmd, true, logout);
      if (rv != 0) {
    // deletion failed -- permissions?
    message = "delpermission";
      } else {
    message = "delsuccess";
      }

      // log the event
      if (rcinfo.LogEvents == CollectorEvents || rcinfo.LogEvents == AllEvents) {
    
    text_t eventlog = filename_cat (gsdlhome, "etc", "events.txt");
    char *eventlogt = eventlog.getcstr();
    ofstream eventl (eventlogt, ios::app);
    delete eventlogt;

    if (eventl) {
      eventl << outconvert << "[Collector Event]\n"
         << "Date: " << get_date (true) << "\n"
         << "Greenstone Username: " << args["un"] << "\n"
         << "Collection: " << collection << "\n"
         << "Collection Maintainer: " << colmaintainer << "\n"
         << "GSDLHOME: " << gsdlhome << "\n";

      if (message == "delsuccess") {
        eventl << outconvert 
           << "The " << collection << " collection was successfully deleted\n\n";
      } else {
        eventl << outconvert 
           << "Attempt to delete the " << collection << " collection failed\n\n";
      }
      eventl.close();

    } else {
      logout << outconvert << "collectoraction::do_action ERROR: Couldn't open "
         << "event log file " << eventlog << " for appending during collection "
         << "deletion. LogEvents disabled\n";
    }
      }
      
      if (rcinfo.EmailEvents == CollectorEvents || rcinfo.EmailEvents == AllEvents || emailuserevents) {
    // use sendmail.pl perl script to send email events
    text_t tmpmailfile = filename_cat (gsdlhome, "tmp", args["bc1tmp"], "event.txt");
    char *tmpmailfilec = tmpmailfile.getcstr();
    ofstream tmpfile (tmpmailfilec);
    delete tmpmailfilec;
    if (tmpfile) {
      tmpfile << outconvert << "[Collector Event]\n"
              << "Date: " << get_date (true) << "\n"
          << "Greenstone Username: " << args["un"] << "\n"
          << "Collection: " << collection << "\n"
          << "Collection Maintainer: " << colmaintainer << "\n"
          << "GSDLHOME: " << gsdlhome << "\n";
      if (message == "delsuccess") {
        tmpfile << outconvert 
            << "The " << collection << " collection was successfully deleted\n\n";
      } else {
        tmpfile << outconvert 
            << "Attempt to delete the " << collection << " collection failed\n\n";
      }
      tmpfile.close();
      text_t to;
      if (rcinfo.EmailEvents == CollectorEvents || rcinfo.EmailEvents == AllEvents) to += rcinfo.maintainer;
      if (emailuserevents) {
        if (!to.empty()) to.push_back (',');
        to += colmaintainer;
      }
      text_t sendmail_cmd = "perl -S sendmail.pl -to \"" + to + "\" -from \"" + rcinfo.maintainer;
      sendmail_cmd += "\" -smtp \"" + rcinfo.MailServer + "\" -subject \"Greenstone Collector Event\"";
      sendmail_cmd += " -msgfile \"" + tmpmailfile + "\"";

      gsdl_system (sendmail_cmd, false, logout);
      
    } else {
      logout << outconvert << "collectoraction::do_action ERROR: Couldn't open "
         << "temporary event log file " << tmpmailfile << " during collection "
         << "deletion. EmailEvents and EmailUserEvents disabled\n";
    }
      }
    }
  }
  
  if (collector_page == "bildcancel" || collector_page == "bildfail") {
    // cancel the build (we'll also use the cancel_build script to tidy
    // up if the build failed)
    gsdl_cancel_build (args, logout);
  }

  if (collector_page == "expt") {

    // export the collection - we'll do a synchronous system call to
    // exportcol.pl as that's the easiest way to do it. if it becomes a
    // problem that it's taking too long to export a large collection then
    // we may have to revisit this.
    text_t tmpfile = filename_cat (gsdlhome, "tmp", collection + "_export.txt");
    text_t export_cmd = "perl -S exportcol.pl -out \"" + tmpfile + "\" " + collection;
    gsdl_system (export_cmd, true, logout);
    if (file_exists (tmpfile)) {
      text_t returnline = file_tail (tmpfile, 1, 0);
      if (returnline.size() > 23 && (substr(returnline.begin(), returnline.begin()+23) == "exportcol.pl succeeded:")) {
    // success
    message = "exptsuccess";
      } else {
    message = "exptfail";
      }
    } else {
      message = "exptfail";
    }
  }

  if (message.empty()) {
    if (collector_page != "bild") {
      // output page ("bild" page was already output above)
      textout << outconvert << disp << ("_collector:header_\n")
          << ("_collector:" + collector_page + "content_\n")
          << ("_collector:footer_\n");
    }
  } else {
    // message was set somewhere (probably an error), output message page
    textout << outconvert << disp << ("_collector:header_\n")
        << ("_collector:" + message + "content_\n")
        << ("_collector:footer_\n");
    message.clear();
  }
  return true;
}

// if sw = 0 replace all carriage returns in intext with the string "\n"
// else replace all occurances of "\n" with a carriage return
text_t collectoraction::carriage_replace (const text_t &intext, int sw) {
   
  text_t outtext;
  text_t::const_iterator here = intext.begin();
  text_t::const_iterator end = intext.end();
  while (here != end) {
    if (sw == 0) {
      if (*here == '\n') {
    if ((here+1) != end && *(here+1) == '\r') here ++;
    outtext += "\\n";
      } else if (*here == '\r') {
    if ((here+1) != end && *(here+1) == '\n') here ++;
    outtext += "\\n";
      } else {
    outtext.push_back (*here);
      }
    } else if (*here == '\\' && (here+1) != end && *(here+1) == 'n') {
      outtext.push_back ('\n');
      here ++;
    } else {
      outtext.push_back (*here);
    }
    here ++;
  }
  return outtext;
}

// create a short directory name from fullname
text_t collectoraction::get_directory_name (const text_t &fullname) {

  text_t shortname;
  if (fullname.empty()) {
    shortname = "coll";

  } else {
    
    // first make all lowercase and remove any dodgy characters
    // (i.e. anything not [a-z]
    text_t::const_iterator here = fullname.begin();
    text_t::const_iterator end = fullname.end();
    while (here != end) {
      if ((*here >= 'A' && *here <= 'Z') || (*here >= 'a' && *here <= 'z') ||
      (*here == ' ')) {
    if (*here >= 'A' && *here <= 'Z') shortname.push_back (*here+32);
    else if (*here == ' ') {
      while ((*(here+1)) == ' ') here ++;
      shortname.push_back (*here);
    } else shortname.push_back (*here);
      }
      here ++;
    }

    text_tarray words;
    splitchar (shortname.begin(), shortname.end(), ' ', words);
    int num_words = words.size();

    if (num_words == 0) {
      shortname = "coll";

    } else {

      shortname.clear();
      int use_words = (num_words <= 6) ? num_words : 6;
      unsigned int substr_len = 6 / use_words;
      
      for (int i = 0; i < use_words; i++) {
    if (words[i].size() < substr_len) shortname += words[i];
    else shortname += substr (words[i].begin(), words[i].begin()+substr_len);
      }
    }
  }

  // check to see if shortname is unique
  text_t fulldirname = filename_cat (gsdlhome, "collect", shortname);
  if (directory_exists (fulldirname)) {
    int version = 0;
    text_t newname;
    do {
      version ++;
      newname = shortname;
      newname.push_back ('v');
      newname.appendint (version);
      fulldirname = filename_cat (gsdlhome, "collect", newname);
    } while (directory_exists (fulldirname));

    shortname = newname;
  }
  
  return shortname;
}

// tests if collection is write protected (currently just checks if
// collect.cfg file is writable
bool collectoraction::collection_protected (const text_t &collection) {
  text_t cfgfile = filename_cat(gsdlhome, "collect", collection, "etc", "collect.cfg");
  if (file_writable(cfgfile)) return false;
  return true;
}

// assigns a temporary directory name for this collector session
// and creates temporary directory
void collectoraction::assign_tmpname (cgiargsclass &args, ostream &logout) {

  int i = 0;
  text_t tmpname = "tbuild";
  while (directory_exists (filename_cat (gsdlhome, "tmp", tmpname + text_t(i)))) {
    i++;
  }
  tmpname.appendint (i);

  text_t fulltmpdir = filename_cat (gsdlhome, "tmp", tmpname);
  if (!mk_dir (fulltmpdir)) {
    outconvertclass text_t2ascii;
    logout << text_t2ascii << "collectoraction::assign_tmpname unable to create directory ("
       << fulltmpdir << ")\n";
  }

  args["bc1tmp"] = tmpname;
}

void collectoraction::gsdl_mkcol (cgiargsclass &args, ostream &logout) {

  text_t tmpdir = filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  if (!directory_exists (tmpdir)) {
    message = "tmpfail";
    return;
  }

  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // check for a .create file - if it exists then we've already created the collection
  text_t createfile = filename_cat (tmpdir, ".create");
  if (file_exists (createfile)) {
    return;
  }

  // set up options
  text_t options = "-quiet -creator \"" + args["bc1contactemail"] + "\"";
  options += " -title \"" + args["bc1fullname"] + "\"";
  options += " -about \"" + carriage_replace (args["bc1aboutdesc"] + "_collectorextra_", 0) + "\"";
  options += " -collectdir \"" + remove_trailing_slashes(tmpdir) + "\" ";

  text_t optionfile = filename_cat (tmpdir, "mkcol.opt");
  char *optionfilec = optionfile.getcstr();
  ofstream ofile_out (optionfilec);
  delete optionfilec;
  if (!ofile_out) {
    message = "tmpfail";
    return;
  }
  outconvertclass text_t2ascii;
  ofile_out << text_t2ascii << options << "\n";
  ofile_out.close();

  // run mkcol.pl
  text_t mkcol_cmd = "perl -S mkcol.pl -optionfile \"" + optionfile;
  mkcol_cmd += "\" " + collection;
  gsdl_system (mkcol_cmd, true, logout);

  // make sure it went ok
  text_t cfgfile = filename_cat (tmpdir, collection, "etc", "collect.cfg");
  if (!file_writable (cfgfile)) {
    message = "mkcolfail";
  } else {
    // create the .create file (this file is just a place holder to let any future
    // pages know that the collection already exists).
    char *createfilec = createfile.getcstr();
    ofstream cfile_out (createfilec);
    delete createfilec;
    if (cfile_out) {
      cfile_out << "collection created\n";
      cfile_out.close();
    } else {
      message = "tmpfail";
      return;
    }
  }
}

void collectoraction::gsdl_build (cgiargsclass &args, ostream &logout) {

  outconvertclass text_t2ascii;

  text_t tmpdir = filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  if (!directory_exists (tmpdir)) {
    message = "tmpfail";
    return;
  }
  
  text_t &collection = args["bc1dirname"];
  if (collection.empty()) {
    message = "nocollection";
    return;
  }

  // check for a .build file - if it exists then we've already built
  // the collection (or are in the process of building it)
  text_t buildfile = filename_cat (tmpdir, ".build");
  if (file_exists (buildfile)) {
    return;
  } else {
    // create the .build file (this file is just a place holder to let any future
    // pages know that we've already been here)
    char *buildfilec = buildfile.getcstr();
    ofstream bfile_out (buildfilec);
    delete buildfilec;
    if (bfile_out) {
      bfile_out << "collection building\n";
      bfile_out.close();
    } else {
      message = "tmpfail";
      return;
    }
  }

  const recptconf &rcinfo = recpt->get_configinfo ();

  // create the event header file if LogEvents, EmailEvents or
  // EmailUserEvents options are turned on.
  bool logevents = 
    (rcinfo.LogEvents == CollectorEvents || rcinfo.LogEvents == AllEvents || 
     rcinfo.EmailEvents == CollectorEvents || rcinfo.EmailEvents == AllEvents ||
     rcinfo.EmailUserEvents);
  text_t ehead_file = filename_cat (tmpdir, "ehead.txt");
  if (logevents) {
    if (!create_event_header_file (ehead_file, args, logout)) {
      logevents = false;
    }
  }
    
  text_t collectdir = get_collectdir (args);

  // set up build options
  text_t options = "-make_writable -remove_import -out \"";
  options += filename_cat (tmpdir, collection + ".bld");
  options += "\" -collectdir \"" + collectdir + "\" -statsfile \"";
  options += filename_cat(collectdir, collection, "etc", "import.log") + "\"";

  if (args["bc1esrce"] == 1) {
    // we're adding data to an existing collection
    options += " -save_archives -append";
  }

  text_tarray inputvalues, inputtypes;
  splitchar (args["bc1input"].begin(), args["bc1input"].end(), ',', inputvalues);
  splitchar (args["bc1inputtype"].begin(), args["bc1inputtype"].end(), ',', inputtypes);
  int numvalues = inputvalues.size();
  int numtypes = inputtypes.size();
  for (int i = 0; i < numvalues; i++) {
    if (!inputvalues[i].empty()) {
      text_t type = "file://"; // default
      if (i < numtypes) type = inputtypes[i];
      options += " -download \"" + 
    remove_trailing_slashes(type + format_url(decode_commas(inputvalues[i]))) + "\"";
    }
  }

  if (logevents) {
    if (rcinfo.LogEvents == CollectorEvents || rcinfo.LogEvents == AllEvents) 
      options += " -log_events";
    if (rcinfo.EmailEvents == CollectorEvents || rcinfo.EmailEvents == AllEvents) {
      options += " -mail_server " + rcinfo.MailServer;
      options += " -email_events " + rcinfo.maintainer;
      if (rcinfo.EmailUserEvents) options += "," + args["bc1contactemail"];
    } else if (rcinfo.EmailUserEvents) {
      options += " -mail_server " + rcinfo.MailServer;
      options += " -email_events " + args["bc1contactemail"];
    }
    options += " -event_header " + ehead_file;
  }

  text_t optionfile = filename_cat (tmpdir, "build.opt");
  char *optionfilec = optionfile.getcstr();
  ofstream ofile_out (optionfilec);
  delete optionfilec;
  if (!ofile_out) {
    message = "tmpfail";
    return;
  }
  ofile_out << text_t2ascii << options << "\n";
  ofile_out.close();

  // if we're altering an existing collection we need to kill off
  // the existing collection server - we do this for the local library
  // (and any other persistent version of the library) as the existing
  // gdbm file can't be deleted while the collection server holds it open
  if ((args["bc1econf"] == 1) || (args["bc1esrce"] == 1)) {
    remove_colservr (collection, logout);
  }    

  // set up the build command - build.bat has some issues with quoting
  // on win2k when gsdlhome contains spaces so we'll avoid using 
  // "perl -S" here in favor of calling the "build" perl script explicitly
  text_t build_cmd = "perl \"" + filename_cat (gsdlhome, "bin", "script", "build");
  build_cmd += "\" -optionfile \"" + optionfile + "\" " + collection;
  // run build command in background (i.e. asynchronously)
  gsdl_system (build_cmd, false, logout);
}

void collectoraction::gsdl_cancel_build (cgiargsclass &args, ostream &logout) {
  // I really wanted to do what this perl script does from within the library
  // c++ code. I ran into some problems though (like how do you write a portable
  // "rm -r" in c++?). One day I'll spend some time sorting it out ... maybe.
  text_t cancel_cmd = "perl -S cancel_build.pl -collectdir \"";
  cancel_cmd += filename_cat (gsdlhome, "tmp", args["bc1tmp"]) + "\" ";
  cancel_cmd += args["bc1dirname"];
  // To be on the safe side we'll make this a synchronous call
  // so that all tidying up is done before the user has a chance
  // to do anything else (like start rebuilding their collection).
  // This means that for a big collection where there's lots of 
  // stuff to delete etc. it might take a while before the "build
  // cancelled" page appears.
  gsdl_system (cancel_cmd, true, logout);
}

text_t collectoraction::get_collectdir (cgiargsclass &args) {

  if ((args["bc1econf"] == 1) || (args["bc1esrce"] == 1)) {
    // we're adding to a collection in place
    return filename_cat(gsdlhome, "collect");  

  } else {
    return filename_cat (gsdlhome, "tmp", args["bc1tmp"]);
  }
}

// checks to see if any of the plugins in pluginset occur in 
// collections configuration file
bool collectoraction::uses_weird_plugin (const text_t &collection) {

  text_tset pluginset;
  pluginset.insert ("HBPlug");

  text_t cfgfile_content;
  text_t cfgfile_name = filename_cat (gsdlhome, "collect", collection, "etc", "collect.cfg");
  text_t pluginstr, pluginname;

  if (read_file (cfgfile_name, cfgfile_content)) {
    text_t::const_iterator here = cfgfile_content.begin();
    text_t::const_iterator end = cfgfile_content.end();
    while (here != end) {
      here = findchar (here, end, 'p');
      if (here == end) break;
      if ((here+6 < end) && (substr (here, here+6) == "plugin")) {
    getdelimitstr (here+6, end, '\n', pluginstr);
    text_t::const_iterator hp = pluginstr.begin();
    text_t::const_iterator ep = pluginstr.end();
    bool found = false;
    // remove any leading whitespace, trailing options etc.
    while (hp != ep) {
      if (*hp == '\t' || *hp == ' ' || *hp == '\n') {
        if (found) break;
      } else {
        pluginname.push_back (*hp);
        found = true;
      }
      hp ++;
    }
    text_tset::const_iterator it = pluginset.find (pluginname);
    if (it != pluginset.end()) return true; // found matching plugin
    pluginname.clear();
      }
      here ++;
    }
  }
  return false;
}

// create and initialize a new collection server and
// add it to the null protocol.
void collectoraction::create_colserver (const text_t &collection, ostream &logout) {

  recptprotolistclass *protos = recpt->get_recptprotolist_ptr();
  recptprotolistclass::iterator rprotolist_here = protos->begin();
  recptprotolistclass::iterator rprotolist_end = protos->end();
  while (rprotolist_here != rprotolist_end) {
    comerror_t err = noError;
    if ((*rprotolist_here).p != NULL) {
      if ((*rprotolist_here).p->get_protocol_name (err) == "nullproto") {
    // create collection server and add it to nullproto
    (*rprotolist_here).p->add_collection (collection, recpt, gsdlhome, gsdlhome);
    // make sure gsdlhome is configured
    text_tarray tmp;
    tmp.push_back (gsdlhome);
    (*rprotolist_here).p->configure ("gsdlhome", tmp, err);
    // re-initialize the null protocol
    if (!(*rprotolist_here).p->init (err, logout)) {
      logout << "collectoraction::create_colserver: nullproto init failed\n";
    }
    return;
      }
    }
    rprotolist_here ++;
  }

  logout << "collectoraction::create_colserver: no valid nullproto found\n";
}

// delete a collection server from the null protocol
void collectoraction::remove_colservr (const text_t &collection, ostream &logout) {

  recpt->uncache_collection (collection);

  recptprotolistclass *protos = recpt->get_recptprotolist_ptr();
  recptprotolistclass::iterator rprotolist_here = protos->begin();
  recptprotolistclass::iterator rprotolist_end = protos->end();
  while (rprotolist_here != rprotolist_end) {
    comerror_t err = noError;
    if ((*rprotolist_here).p != NULL) {
      if ((*rprotolist_here).p->get_protocol_name (err) == "nullproto") {
    (*rprotolist_here).p->remove_collection (collection, logout);
    return;
      }
    }
    rprotolist_here ++;
  }

  logout << "collectoraction::create_colserver: no valid nullproto found\n";
}

bool collectoraction::create_event_header_file (const text_t &filename, cgiargsclass &args, 
                        ostream &logout) {

  outconvertclass text_t2ascii;
  char *filenamec = filename.getcstr();
  ofstream eheadfile (filenamec);
  delete filenamec;

  if (eheadfile) {
    eheadfile << text_t2ascii << get_event_header (args);
    eheadfile.close();
    return true;
  }
   
  logout << text_t2ascii << "collectoraction::create_event_header ERROR: Couldn't create "
     << "Event Header file " << filename << ". Event logging disabled\n";
  return false;
}

text_t collectoraction::get_event_header (cgiargsclass &args) {
  text_t header = "Greenstone Username: " + args["un"] + "\n";
  header += "Collection: " + args["bc1dirname"] + "\n";
  header += "Collection Creator: " + args["bc1contactemail"] + "\n";
  header += "GSDLHOME: " + gsdlhome + "\n";
  header += "Build Location: " + get_collectdir(args) + "\n";

  return header;
}

bool collectoraction::check_sources (cgiargsclass &args, ostream &logout) {

  bool found = false;

  text_tarray inputvalues;
  splitchar (args["bc1input"].begin(), args["bc1input"].end(), ',', inputvalues);

  text_tarray inputtypes;
  splitchar (args["bc1inputtype"].begin(), args["bc1inputtype"].end(), ',', inputtypes);

  int numvalues = inputvalues.size();
  int numtypes = inputtypes.size();

  for (int i = 0; i < numvalues; i++) {
    text_t value = format_url(decode_commas(inputvalues[i]));
    text_t type = "file://"; // default
    if (!value.empty()) {
      found = true;
      if (i >= numtypes || inputtypes[i].empty()) {
    logout << "collectoraction::check_sources: WARNING type not set\n";
      } else {
    type = inputtypes[i];
      }
      if (type == "file://") {
    if (!file_exists(value) && !directory_exists(value)) {
      failedsources[decode_commas(inputvalues[i])] = "1";
      badsources = true;
    }
      } else if (type == "http://") {
    if (gsdl_system ("perl -S ping.pl -quiet http://" + value, true, logout)) {
      failedsources[decode_commas(inputvalues[i])] = "1";
      badsources = true;
    }
      } else if (type == "ftp://") {
    if (gsdl_system ("perl -S ping.pl -quiet ftp://" + value, true, logout)) {
      failedsources[decode_commas(inputvalues[i])] = "1";
      badsources = true;
    }
      }
    }
  }

  // set badsources if there weren't any sources at all
  if (!found) badsources = true;

  if (badsources) return false;
  return true;
}

// format_url simply strips "http://", "ftp://", or "file://" off the
// beginning of url if they're there
text_t collectoraction::format_url (const text_t &url) {
  text_t::const_iterator begin = url.begin();
  text_t::const_iterator end = url.end();

  if (url.size() >= 7) {
    text_t prefix = substr(begin, begin+7);
    if (prefix == "http://" || prefix == "file://") {
      return substr(begin+7, end);
    }
  }
  if (url.size() >= 6) {
    if (substr(begin, begin+6) == "ftp://") {
      return substr(begin+6, end);
    }
  }
  return url;
}

text_t collectoraction::remove_trailing_slashes (text_t str) {

  while (*(str.end()-1) == '\\') {
    str.pop_back();
  }
  return str;
}