package org.greenstone.gsdl3.core;

import java.io.File;
import java.io.FileReader;
import java.io.Serializable;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.xerces.parsers.DOMParser;
import org.greenstone.gsdl3.action.Action;
import org.greenstone.gsdl3.util.GSConstants;
import org.greenstone.gsdl3.util.GSFile;
import org.greenstone.gsdl3.util.GSParams;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.GSXSLT;
import org.greenstone.gsdl3.util.UserContext;
import org.greenstone.gsdl3.util.XMLConverter;
import org.greenstone.gsdl3.util.XMLTransformer;
import org.greenstone.gsdl3.util.XSLTUtil;
import org.greenstone.util.GlobalProperties;
import org.w3c.dom.Comment;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.InputSource;

/**
 * A receptionist that uses xslt to transform the page_data before returning it.
 * . Receives requests consisting of an xml representation of cgi args, and
 * returns the page of data - in html by default. The requests are processed by
 * the appropriate action class
 * 
 * @see Action
 */
public class TransformingReceptionist extends Receptionist
{
  protected static final String EXPAND_GSF_FILE = "expand-gsf.xsl"; 
  protected static final String EXPAND_GSF_PASS1_FILE = "expand-gsf-pass1.xsl"; 
  protected static final String EXPAND_GSLIB_FILE = "expand-gslib.xsl";
  protected static final String GSLIB_FILE = "gslib.xsl";
  
  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.core.TransformingReceptionist.class.getName());

	/** The expand-gslib.xsl file is in a fixed location */
	static final String expand_gslib_filepath = GlobalProperties.getGSDL3Home() + File.separatorChar + "interfaces" + File.separatorChar + "core" + File.separatorChar + "transform" + File.separatorChar + EXPAND_GSLIB_FILE;

	/** the list of xslt to use for actions */
	protected HashMap<String, String> xslt_map = null;

	/** a transformer class to transform xml using xslt */
	protected XMLTransformer transformer = null;

	protected TransformerFactory transformerFactory = null;
	protected DOMParser parser = null;

	protected HashMap<String, ArrayList<String>> _metadataRequiredMap = new HashMap<String, ArrayList<String>>();

	boolean _debug = true;

	public TransformingReceptionist()
	{
		super();
		this.xslt_map = new HashMap<String, String>();
		this.transformer = new XMLTransformer();
		try
		{
			transformerFactory = org.apache.xalan.processor.TransformerFactoryImpl.newInstance();
			this.converter = new XMLConverter();
			//transformerFactory.setURIResolver(new MyUriResolver()) ;

			parser = new DOMParser();
			parser.setFeature("http://xml.org/sax/features/validation", false);
			// don't try and load external DTD - no need if we are not validating, and may cause connection errors if a proxy is not set up.
			parser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
			// a performance test showed that having this on lead to increased 
			// memory use for small-medium docs, and not much gain for large 
			// docs.
			// http://www.sosnoski.com/opensrc/xmlbench/conclusions.html
			parser.setFeature("http://apache.org/xml/features/dom/defer-node-expansion", false);
			parser.setFeature("http://apache.org/xml/features/continue-after-fatal-error", true);
			// setting a handler for when fatal errors, errors or warnings happen during xml parsing
			// call XMLConverter's getParseErrorMessage() to get the errorstring that can be rendered as web page
			this.parser.setErrorHandler(new XMLConverter.ParseErrorHandler());
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
	}

	/** configures the receptionist - adding in setting up the xslt map */
	public boolean configure()
	{
	    if (!super.configure()) {
		
		return false;
	    }
		
	    logger.info("configuring the TransformingReceptionist");

		// find the config file containing a list of actions
		File interface_config_file = new File(GSFile.interfaceConfigFile(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.INTERFACE_NAME))));
		Document config_doc = this.converter.getDOM(interface_config_file, "UTF-8");
		Element config_elem = config_doc.getDocumentElement();
		
		// Find the actions again so that we can set up the xslt map
		Element action_list = (Element) GSXML.getChildByTagName(config_elem, GSXML.ACTION_ELEM + GSXML.LIST_MODIFIER);
		NodeList actions = action_list.getElementsByTagName(GSXML.ACTION_ELEM);

		for (int i = 0; i < actions.getLength(); i++)
		{
			Element action = (Element) actions.item(i);
			String class_name = action.getAttribute("class");
			String action_name = action.getAttribute("name");

			// now do the xslt map
			String xslt = action.getAttribute("xslt");
			if (!xslt.equals(""))
			{
				this.xslt_map.put(action_name, xslt);
			}
			NodeList subactions = action.getElementsByTagName(GSXML.SUBACTION_ELEM);
			for (int j = 0; j < subactions.getLength(); j++)
			{
				Element subaction = (Element) subactions.item(j);
				String subname = subaction.getAttribute(GSXML.NAME_ATT);
				String subxslt = subaction.getAttribute("xslt");

				String map_key = action_name + ":" + subname;
				logger.debug("adding in to xslt map, " + map_key + "->" + subxslt);
				this.xslt_map.put(map_key, subxslt);
			}
		}

		getRequiredMetadataNamesFromXSLFiles();

		return true;
	}

	protected void getRequiredMetadataNamesFromXSLFiles()
	{
		ArrayList<File> xslFiles = GSFile.getAllXSLFiles((String) this.config_params.get(GSConstants.SITE_NAME));

		HashMap<String, ArrayList<String>> includes = new HashMap<String, ArrayList<String>>();
		HashMap<String, ArrayList<File>> files = new HashMap<String, ArrayList<File>>();
		HashMap<String, ArrayList<String>> metaNames = new HashMap<String, ArrayList<String>>();

		//First exploratory pass
		for (File currentFile : xslFiles)
		{

		    String full_filename = currentFile.getPath();
		    int sep_pos = full_filename.lastIndexOf(File.separator)+1;
		    String local_filename = full_filename.substring(sep_pos);
		    if (local_filename.startsWith(".")) {
			logger.warn("Greenstone does not normally rely on 'dot' files for XSL transformations.\n Is the following file intended to be part of the digital library installation?\n XSL File being read in:\n    " + currentFile.getPath());
		    }
				
			Document currentDoc = this.converter.getDOM(currentFile);
			if (currentDoc == null)
			{
				// Can happen if an editor creates an auto-save temporary file 
				// (such as #header.xsl#) that is not well formed XML
				logger.warn("Skipping XSL file.	 DOM returned was null for:\n	 " + currentFile.getPath());
				continue;
			}

			HashSet<String> extra_meta_names = new HashSet<String>();
			GSXSLT.findExtraMetadataNames(currentDoc.getDocumentElement(), extra_meta_names);
			ArrayList<String> names = new ArrayList<String>(extra_meta_names);			

			metaNames.put(currentFile.getAbsolutePath(), names);

			NodeList includeElems = currentDoc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "include");
			NodeList importElems = currentDoc.getElementsByTagNameNS(GSXML.XSL_NAMESPACE, "import");


			ArrayList<String> includeAndImportList = new ArrayList<String>();
			for (int i = 0; i < includeElems.getLength(); i++)
			{
				includeAndImportList.add(((Element) includeElems.item(i)).getAttribute(GSXML.HREF_ATT));
			}
			for (int i = 0; i < importElems.getLength(); i++)
			{
				includeAndImportList.add(((Element) importElems.item(i)).getAttribute(GSXML.HREF_ATT));
			}
			includes.put(currentFile.getAbsolutePath(), includeAndImportList);

			String filename = currentFile.getName();
			if (files.get(filename) == null)
			{
				ArrayList<File> fileList = new ArrayList<File>();
				fileList.add(currentFile);
				files.put(currentFile.getName(), fileList);
			}
			else
			{
				ArrayList<File> fileList = files.get(filename);
				fileList.add(currentFile);
			}
		}

		//Second pass
		for (File currentFile : xslFiles)
		{
			ArrayList<File> filesToGet = new ArrayList<File>();
			filesToGet.add(currentFile);

			ArrayList<String> fullNameList = new ArrayList<String>();

			while (filesToGet.size() > 0)
			{
				File currentFileTemp = filesToGet.remove(0);

				//Add the names from this file
				ArrayList<String> currentNames = metaNames.get(currentFileTemp.getAbsolutePath());
				if (currentNames == null)
				{
					continue;
				}

				fullNameList.addAll(currentNames);

				ArrayList<String> includedHrefs = includes.get(currentFileTemp.getAbsolutePath());

				for (String href : includedHrefs)
				{
					int lastSepIndex = href.lastIndexOf("/");
					if (lastSepIndex != -1)
					{
						href = href.substring(lastSepIndex + 1);
					}

					ArrayList<File> filesToAdd = files.get(href);
					if (filesToAdd != null)
					{
						filesToGet.addAll(filesToAdd);
					}
				}
			}

			_metadataRequiredMap.put(currentFile.getAbsolutePath(), fullNameList);
		}
	}

	protected void preProcessRequest(Element request)
	{
		String action = request.getAttribute(GSXML.ACTION_ATT);
		String subaction = request.getAttribute(GSXML.SUBACTION_ATT);

		String name = null;
		if (!subaction.equals(""))
		{
			String key = action + ":" + subaction;
			name = this.xslt_map.get(key);
		}
		// try the action by itself
		if (name == null)
		{
			name = this.xslt_map.get(action);
		}

		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
		String collection = "";

		if (cgi_param_list != null)
		{
			// Don't waste time getting all the parameters
			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
			collection = (String) params.get(GSParams.COLLECTION);
			if (collection == null)
			{
				collection = "";
			}
		}

		ArrayList<File> stylesheets = GSFile.getStylesheetFiles(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, name);

		Document doc = XMLConverter.newDOM();
		Element extraMetadataList = doc.createElement(GSXML.EXTRA_METADATA + GSXML.LIST_MODIFIER);
		HashSet<String> name_set = new HashSet<String>();
		for (File stylesheet : stylesheets)
		{
			ArrayList<String> requiredMetadata = _metadataRequiredMap.get(stylesheet.getAbsolutePath());

			if (requiredMetadata != null)
			{
				for (String metadataString : requiredMetadata)
				{
				  if (!name_set.contains(metadataString)) {
				      name_set.add(metadataString);
					Element metadataElem = doc.createElement(GSXML.EXTRA_METADATA);
					metadataElem.setAttribute(GSXML.NAME_ATT, metadataString);
					extraMetadataList.appendChild(metadataElem);
				    }
				}
			}
	}
		request.appendChild(request.getOwnerDocument().importNode(extraMetadataList, true));
	}

	protected Node postProcessPage(Element page)
	{
		// might need to add some data to the page
		addExtraInfo(page);

		
		// transform the page using xslt

		String currentInterface = (String) config_params.get(GSConstants.INTERFACE_NAME);

		Element request = (Element) GSXML.getChildByTagName(page, GSXML.PAGE_REQUEST_ELEM);
		String output = request.getAttribute(GSXML.OUTPUT_ATT);

		boolean useClientXSLT = (Boolean) config_params.get(GSConstants.USE_CLIENT_SIDE_XSLT);
		//logger.info("Client side transforms allowed? " + allowsClientXSLT);

		if (useClientXSLT)
		{
		    // if not specified, output defaults to 'html', but this isn't what we want when useClientXSLT is on
		    if (output.equals("html")) {
			output = "xsltclient";
		    }
		}
		Node transformed_page = transformPage(page,currentInterface,output);

		if (useClientXSLT) {
		    return transformed_page;
		}
		// if the user has specified they want only a part of the full page then subdivide it
		boolean subdivide = false;
		String excerptID     = null;
		String excerptIDText = null;
		String excerptTag    = null;
		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);

		// **** Now that the number of cases handled has risen to 3, the following would be worth refactoring ****
		if (cgi_param_list != null)
		{
			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
			if ((excerptID = (String) params.get(GSParams.EXCERPT_ID)) != null)
			{
				subdivide = true;
			}
			if ((excerptIDText = (String) params.get(GSParams.EXCERPT_ID_TEXT)) != null)
			{
				subdivide = true;
			}
			if ((excerptTag = (String) params.get(GSParams.EXCERPT_TAG)) != null)
			{
				subdivide = true;
			}
		}

		if (subdivide)
		{
		        Node subdivided_page = subdivide(transformed_page, excerptID, excerptIDText, excerptTag);
			if (subdivided_page != null)
			{
				return subdivided_page;
			}
			else return null;
		}

		return transformed_page;
	}

        protected Node subdivide(Node transformed_page, String excerptID, String excerptIDText, String excerptTag)
	{
		if (excerptID != null)
		{
			Node selectedElement = getNodeByIdRecursive(transformed_page, excerptID);
			modifyNodesByTagRecursive(selectedElement, "a");
			return selectedElement;
		}
		if (excerptIDText != null)
		{
			Node selectedElement = getNodeByIdRecursive(transformed_page, excerptIDText);

			String selectedTextString = selectedElement.getTextContent();
			Document forexcerptid_doc = XMLConverter.newDOM();
			Node selectedElementChildTextNode = forexcerptid_doc.createTextNode(selectedTextString);

			return selectedElementChildTextNode;
		}
		else if (excerptTag != null)
		{
			Node selectedElement = getNodeByTagRecursive(transformed_page, excerptTag);
			return selectedElement;
		}
		return transformed_page;
	}

	protected Node getNodeByIdRecursive(Node parent, String id)
	{
		if (parent.hasAttributes() && ((Element) parent).getAttribute("id").equals(id))
		{
			return parent;
		}

		NodeList children = parent.getChildNodes();
		for (int i = 0; i < children.getLength(); i++)
		{
			Node result = null;
			if ((result = getNodeByIdRecursive(children.item(i), id)) != null)
			{
				return result;
			}
		}
		return null;
	}

	protected Node getNodeByTagRecursive(Node parent, String tag)
	{
		if (parent.getNodeType() == Node.ELEMENT_NODE && ((Element) parent).getTagName().equals(tag))
		{
			return parent;
		}

		NodeList children = parent.getChildNodes();
		for (int i = 0; i < children.getLength(); i++)
		{
			Node result = null;
			if ((result = getNodeByTagRecursive(children.item(i), tag)) != null)
			{
				return result;
			}
		}
		return null;
	}

	protected Node modifyNodesByTagRecursive(Node parent, String tag)
	{
		if (parent == null || (parent.getNodeType() == Node.ELEMENT_NODE && ((Element) parent).getTagName().equals(tag)))
		{
			return parent;
		}

		NodeList children = parent.getChildNodes();
		for (int i = 0; i < children.getLength(); i++)
		{
			Node result = null;
			if ((result = modifyNodesByTagRecursive(children.item(i), tag)) != null)
			{
				//TODO: DO SOMETHING HERE?
			}
		}
		return null;
	}

        protected void replaceNodeWithInterfaceText(Document doc, String interface_name, String lang,
						    Element elem, String attr_name, String attr_val)
        {
	    String pattern_str_3arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*\\)";
	    String pattern_str_4arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*,\\s*(.+?)\\s*\\)$";
    
	    Pattern pattern3 = Pattern.compile(pattern_str_3arg);
	    Matcher matcher3 = pattern3.matcher(attr_val);
	    if (matcher3.find()) {
		String dict_key = matcher3.group(1);
		String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);
		
		Node parent_node = elem.getParentNode();

		Text replacement_text_node = doc.createTextNode(dict_val);
	        parent_node.replaceChild(replacement_text_node,elem);
	    }	    
	    else {
		Pattern pattern4 = Pattern.compile(pattern_str_4arg);
		Matcher matcher4 = pattern4.matcher(attr_val);
		StringBuffer string_buffer4 = new StringBuffer();

		if (matcher4.find()) {
		    String dict_key = matcher4.group(1);
		    String args     = matcher4.group(2);
		    args = args.replaceAll("\\$","\\\\\\$");
		    
		    String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);

		    matcher4.appendReplacement(string_buffer4, "js:getInterfaceTextSubstituteArgs('"+dict_val+"',string("+args+"))");
		    matcher4.appendTail(string_buffer4);

		    attr_val = string_buffer4.toString();
		    elem.setAttribute(attr_name,attr_val);
		}
		else {
		    logger.error("Failed to find match in attribute: " + attr_name + "=\"" + attr_val + "\"");
		    attr_val = attr_val.replaceAll("util:getInterfaceText\\(.+?,.+?,\\s*(.+?)\\s*\\)","$1");
		    elem.setAttribute(attr_name,attr_val);
		}
	    }
	
	}
    
        protected void resolveExtendedNamespaceAttributesXSLT(Document doc, String interface_name, String lang)
        {
	    String[] attr_list = new String[] {"select","test"};

	    // http://stackoverflow.com/questions/13220520/javascript-replace-child-loop-issue
	    // go through nodeList in reverse to avoid the 'skipping' problem, due to
	    // replaceChild() calls removing items from the "live" nodeList
	    
	    NodeList nodeList = doc.getElementsByTagName("*");
	    for (int i=nodeList.getLength()-1; i>=0; i--) {
		Node node = nodeList.item(i);
		if (node.getNodeType() == Node.ELEMENT_NODE) {
		    Element elem = (Element)node;		
		    for (String attr_name : attr_list) {
			if (elem.hasAttribute(attr_name)) {
			    String attr_val = elem.getAttribute(attr_name);
			    
			    if (attr_val.startsWith("util:getInterfaceText(")) {
				// replace the node with dictionary lookup
				replaceNodeWithInterfaceText(doc, interface_name,lang, elem,attr_name,attr_val);
			    }							
			    else if (attr_val.contains("util:")) {

				attr_val = attr_val.replaceAll("util:getInterfaceStringsAsJavascript\\(.+?,.+?,\\s*(.+?)\\)","$1");

				//attr_val = attr_val.replaceAll("util:escapeNewLinesAndQuotes\\(\\s*(.+?)\\s*\\)","'escapeNLandQ $1'");
				//attr_val = attr_val.replaceAll("util:escapeNewLinesAndQuotes\\(\\s*(.+?)\\s*\\)","$1");					

				// 'contains()' supported in XSLT 1.0, so OK to change any util:contains() into contains()
				attr_val = attr_val.replaceAll("util:(contains\\(.+?\\))","$1");

				elem.setAttribute(attr_name,attr_val);
			    }

			    if (attr_val.contains("java:")) {
				if (attr_val.indexOf("getInterfaceTextSubstituteArgs")>=4) {
				    
				    attr_val = attr_val.replaceAll("java:.+?\\.(\\w+)\\((.*?)\\)$","js:$1($2)");
				}
				
				elem.setAttribute(attr_name,attr_val);
			    }
			}
		    }
		    
		}
	    }
	}


        protected void resolveExtendedNamespaceAttributesXML(Document doc, String interface_name, String lang)
        {
	    String[] attr_list = new String[] {"src", "href"};

	    // http://stackoverflow.com/questions/13220520/javascript-replace-child-loop-issue
	    // go through nodeList in reverse to avoid the 'skipping' problem, due to
	    // replaceChild() calls removing items from the "live" nodeList
	    
	    NodeList nodeList = doc.getElementsByTagName("*");
	    for (int i=nodeList.getLength()-1; i>=0; i--) {
		Node node = nodeList.item(i);
		if (node.getNodeType() == Node.ELEMENT_NODE) {
		    Element elem = (Element)node;
		    for (String attr_name : attr_list) {
			if (elem.hasAttribute(attr_name)) {
			    String attr_val = elem.getAttribute(attr_name);

			    if (attr_val.contains("util:getInterfaceText(")) {
				String pattern_str_3arg = "util:getInterfaceText\\([^,]+,[^,]+,\\s*'(.+?)'\\s*\\)";
				Pattern pattern3 = Pattern.compile(pattern_str_3arg);
				Matcher matcher3 = pattern3.matcher(attr_val);
				
				StringBuffer string_buffer3 = new StringBuffer();
				
				boolean found_match = false;
				
				while (matcher3.find()) {
				    found_match = true;
				    String dict_key = matcher3.group(1);
				    String dict_val = XSLTUtil.getInterfaceText(interface_name,lang,dict_key);
				    
				    matcher3.appendReplacement(string_buffer3, dict_val);
				}
				matcher3.appendTail(string_buffer3);
				
				if (found_match) {
				    attr_val = string_buffer3.toString();
				    elem.setAttribute(attr_name,attr_val);				    
				}
				else {			    
				    logger.error("Failed to find match in attribute: " + attr_name + "=\"" + attr_val + "\"");
				    attr_val = attr_val.replaceAll("util:getInterfaceText\\(.+?,.+?,\\s*(.+?)\\s*\\)","$1");
				    elem.setAttribute(attr_name,attr_val);
				}
			    }
			    else if (attr_val.contains("util:")) {

				logger.error("Encountered unexpected 'util:' prefix exension: " + attr_name + "=\"" + attr_val + "\"");
			    }

			    if (attr_val.contains("java:")) {
				// make anything java: safe from the point of an XSLT without extensions
				logger.error("Encountered unexpected 'java:' prefix exension: " + attr_name + "=\"" + attr_val + "\"");

			    }
			}
		    }
		    
		}
	    }
	}

    
    
	/**
	 * overwrite this to add any extra info that might be needed in the page
	 * before transformation
	 */
	protected void addExtraInfo(Element page)
	{
	}

	/**
	 * transform the page using xslt.
	 * we need to get any format element out of the page and add it to the xslt before transforming
	 */
        protected Node transformPage(Element page_xml, String currentInterface, String output)
	{
		_debug = false;

		Element request = (Element) GSXML.getChildByTagName(page_xml, GSXML.PAGE_REQUEST_ELEM);

		//logger.info("Current output mode is: " + output + ", current interface name is: " + currentInterface);
		
		if (output.equals("xsltclient"))
		{
		  return generateXSLTClientOutput(request);
		}
		
		String action = request.getAttribute(GSXML.ACTION_ATT);
		String subaction = request.getAttribute(GSXML.SUBACTION_ATT);

		// we should choose how to transform the data based on output, eg diff
		// choice for html, and wml??
		// for now, if output=xml, we don't transform the page, we just return 
		// the page xml
		Document page_with_xslt_params_doc = null;

		if (output.equals("xml") || (output.equals("json")) || output.equals("clientside"))
		{
			// Append the xsltparams to the page
		        page_with_xslt_params_doc = converter.newDOM();
			// Import into new document first!
			Node page_with_xslt_params = page_with_xslt_params_doc.importNode(page_xml, true);
			page_with_xslt_params_doc.appendChild(page_with_xslt_params);
			Element xslt_params = page_with_xslt_params_doc.createElement("xsltparams");
			page_with_xslt_params.appendChild(xslt_params);

			GSXML.addParameter2ToList(xslt_params, "library_name", (String) config_params.get(GSConstants.LIBRARY_NAME));
			GSXML.addParameter2ToList(xslt_params, "interface_name", (String) config_params.get(GSConstants.INTERFACE_NAME));
			GSXML.addParameter2ToList(xslt_params, "site_name", (String) config_params.get(GSConstants.SITE_NAME));
			GSXML.addParameter2ToList(xslt_params, "cookie_consent", (String) config_params.get(GSConstants.COOKIE_CONSENT));
			Boolean useClientXSLT = (Boolean) config_params.get(GSConstants.USE_CLIENT_SIDE_XSLT);
			GSXML.addParameter2ToList(xslt_params, "use_client_side_xslt", useClientXSLT.toString());
			GSXML.addParameter2ToList(xslt_params, "filepath", GlobalProperties.getGSDL3Home());
			
			if ((output.equals("xml")) || output.equals("json"))
			{
			  // Just return the page XML
			  // in the case of "json", calling method responsible for converting to JSON-string
			  return page_with_xslt_params_doc.getDocumentElement();
			}
			// in the case of client side, later on we'll use this doc with xslt params added,
			// along with the xsl.
		}

		Element cgi_param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
		String collection = "";
		String inline_template = "";
		if (cgi_param_list != null)
		{
			// Don't waste time getting all the parameters
			HashMap<String, Serializable> params = GSXML.extractParams(cgi_param_list, false);
			collection = (String) params.get(GSParams.COLLECTION);
			if (collection == null)
			{
				collection = "";
			}

			inline_template = (String) params.get(GSParams.INLINE_TEMPLATE);
			String debug_p = (String) params.get(GSParams.DEBUG);
			if (debug_p != null && (debug_p.equals("on") || debug_p.equals("1") || debug_p.equals("true")))
			{
				String[] groups = new UserContext(request).getGroups();

				boolean found = false;
				for (String g : groups)
				{
					if (g.equals("administrator"))
					{
						found = true;
						break;
					}
					if (!collection.equals("")) {
					  if (g.equals("all-collections-editor")) {
					    found = true;
					    break;
					  }
					 
					  if (g.equals(collection+"-collection-editor")) {
					    found = true;
					    break;
					  }
					}
				}
				if (found)
				{
					_debug = true;
				}
			}
		}

		config_params.put("collName", collection);

		// find the appropriate stylesheet (based on action/subaction) - eg a=p&sa=home will be home.xsl
		// This mapping is defined in interfaceConfig.xsl
		// All versions of the stylesheet (base interface, interface, site, collection) are
		// merged together into one document
		Document page_xsl = getXSLTDocument(action, subaction, collection);
		String page_xsl_filename = getXSLTFilename(action, subaction); // for debug purposes
		if (page_xsl == null)
		{
		  logger.error("Couldn't find and/or load the stylesheet ("+page_xsl_filename+") for a="+action+", sa="+subaction+", in collection="+collection);
		  return XMLTransformer.constructErrorXHTMLPage("Couldn't find and/or load the stylesheet \""+page_xsl_filename+"\" for a="+action+", sa="+subaction+", in collection="+collection);
		}

		if (output.equals("xsl1")) {
		  // if we just output the page_xsl directly then there may be unescaped & in the javascript,
		  // and the page won't display properly
		  return converter.getDOM(getStringFromDocument(page_xsl));
		}

		
		// put the page into a document - this is necessary for xslt to get
		// the paths right if you have paths relative to the document root
		// eg /page.
		Document page_xml_doc = XMLConverter.newDOM();
		page_xml_doc.appendChild(page_xml_doc.importNode(page_xml, true));
		Element page_response = (Element) GSXML.getChildByTagName(page_xml, GSXML.PAGE_RESPONSE_ELEM);
		Element format_elem = (Element) GSXML.getChildByTagName(page_response, GSXML.FORMAT_ELEM);

		if (output.equals("format1"))
		{
			return format_elem;
		}

		// do we have language attribute for page?
		String lang_att = page_xml.getAttribute(GSXML.LANG_ATT);
		if (lang_att != null && lang_att.length() > 0)
		  {
		    config_params.put("lang", lang_att);
		  }
		
		if (format_elem != null)
		{
		  //page_response.removeChild(format_elem);
		  
		  // need to transform the format info
		  // run expand-gsf.xsl over the format_elem. We need to do this now to turn
		  // eg gsf:template into xsl:template so that the merging works properly.
		  // xsl:templates will get merged
		  // into the main stylesheet, but gsf:templates won't.

		  Document format_doc = XMLConverter.newDOM();
		  format_doc.appendChild(format_doc.importNode(format_elem, true));
		  
		  if (_debug) {
		    
		    String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME));
		    GSXSLT.insertDebugElements(format_doc, GSFile.collectionConfigFile(siteHome, collection));
		  }
		  
		  // should we be doing the double pass here too?
		  Node result = transformGSFElements(collection, format_doc, EXPAND_GSF_FILE);
		  // Since we started creating documents with DocTypes, we can end up with 
		  // Document objects here. But we will be working with an Element instead, 
		  // so we grab the DocumentElement() of the Document object in such a case.
				Element new_format;
				if (result.getNodeType() == Node.DOCUMENT_NODE)
				{
					new_format = ((Document) result).getDocumentElement();
				}
				else
				{
					new_format = (Element) result;
				}
				
				if (output.equals("format"))
				{
					return new_format;
				}

				// add the extracted format statements in to the main stylesheet
				if (_debug)
				{
					String siteHome = GSFile.siteHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME));
					GSXSLT.mergeStylesheetsDebug(page_xsl, new_format, true, true, "OTHER1", GSFile.collectionConfigFile(siteHome, collection));
				}
				else
				{
				  GSXSLT.mergeStylesheets(page_xsl, new_format, true);
				}

		
		}

		if (output.equals("xsl2")) {
		  return converter.getDOM(getStringFromDocument(page_xsl));
		}

		Document inline_template_doc = null;
		if (inline_template != null)
		{
			try
			{
			  inline_template_doc = this.converter.getDOM("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<xsl:stylesheet version=\"1.0\" "+GSXML.ALL_NAMESPACES_ATTS +  ">" + inline_template + "</xsl:stylesheet>", "UTF-8");

				if (_debug)
				{
					GSXSLT.mergeStylesheetsDebug(page_xsl, inline_template_doc.getDocumentElement(), true, true, "OTHER2", "INLINE");
				}
				else
				{
				  //GSXSLT.mergeStylesheets(skinAndLibraryDoc, inlineTemplateDoc.getDocumentElement(), true);
				  GSXSLT.mergeStylesheets(page_xsl, inline_template_doc.getDocumentElement(), true);
				}
			}
			catch (Exception ex)
			{
				ex.printStackTrace();
			}
		}

		
		if (output.equals("ilt")) {
		  return converter.getDOM(getStringFromDocument(inline_template_doc));
		}
		if (output.equals("xsl3")) {
		  return converter.getDOM(getStringFromDocument(page_xsl));
		}

		// once we are here, have got the main page xsl loaded up. Have added in any format statements from the source xml, and added in any inline template which came through cgi params.
		
		// next we load in the import and include files. - these, too, go through the inheritance cascade (base interface, interface, site, collection) before being added into the main document
		
		if (_debug)
		{
		  GSXSLT.inlineImportAndIncludeFilesDebug(page_xsl, null, _debug, page_xsl_filename, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
		}
		else
		{
		  GSXSLT.inlineImportAndIncludeFiles(page_xsl, null, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
		
		}

		if (output.equals("xsl4")) {
		  return converter.getDOM(getStringFromDocument(page_xsl));
		}
		
		// The next step is to process the page_xsl +  gslib_xsl by
		// expand-gslib.xsl to expand all the gslib elements

                Document expand_gslib_xsl_doc;
		try
		{
		  // interfaces/core/transform/expand-gslib.xsl
		  // this takes skinandLibraryXsl, copies skinXSL, merges elements of libraryXsl into it, and replaces gslib elements
			expand_gslib_xsl_doc = getDoc(expand_gslib_filepath);
			String errMsg = ((XMLConverter.ParseErrorHandler) parser.getErrorHandler()).getErrorMessage();
			if (errMsg != null)
			{
				return XMLTransformer.constructErrorXHTMLPage("error loading file: " + expand_gslib_filepath + "\n" + errMsg);
			}
		}
		catch (java.io.FileNotFoundException e)
		{
			return fileNotFoundErrorPage(e.getMessage());
		}
		catch (Exception e)
		{
			e.printStackTrace();
			System.out.println("error loading "+expand_gslib_filepath);
			return XMLTransformer.constructErrorXHTMLPage("Error loading file: "+ expand_gslib_filepath+"\n" + e.getMessage());
		}
               
		// gslib.xsl
		Document gslib_xsl_doc = null;
		try
		{
			gslib_xsl_doc = GSXSLT.mergedXSLTDocumentCascade(GSLIB_FILE, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, _debug);
		}
		catch (Exception e)
		{
			e.printStackTrace();
			System.out.println("error loading gslib xslt");
			return XMLTransformer.constructErrorXHTMLPage("error loading gslib xslt\n" + e.getMessage());
		}

  if (output.equals("gslib-expander")) {
    return converter.getDOM(getStringFromDocument(expand_gslib_xsl_doc));
  }
  if (output.equals("gslib1")) {
    return converter.getDOM(getStringFromDocument(gslib_xsl_doc));
  }
		//   Combine the skin file and library variables/templates into one document. 
		//   Please note: We dont just use xsl:import because the preprocessing stage  
		//   needs to know what's available in the library.


                // add in all gslib.xsl's include and import files
		// debug?? use debug method?? or does it not make sense here??
		GSXSLT.inlineImportAndIncludeFiles(gslib_xsl_doc, null, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces);
  if (output.equals("gslib")) {
    return converter.getDOM(getStringFromDocument(gslib_xsl_doc));
  }

  // if the page xsl or gslib xsl uses namespaces that are not listed in
  // expand_gslib, then they will be ignored. So just check through and add
  // any in that are missing.
  GSXML.addMissingNamespaceAttributes(expand_gslib_xsl_doc.getDocumentElement(), page_xsl.getDocumentElement());
  GSXML.addMissingNamespaceAttributes(expand_gslib_xsl_doc.getDocumentElement(), gslib_xsl_doc.getDocumentElement());

  		Document pageAndGslibXsl = null;
		Document pageAndGslibDoc = converter.newDOM();

  // now, we transform all the gslib elements
		{
		  

		  pageAndGslibXsl = converter.newDOM();
		  Element root = pageAndGslibXsl.createElement("pageAndGslibXsl");
		  pageAndGslibXsl.appendChild(root);
		  
		  Element s = pageAndGslibXsl.createElement("pageXsl");
		  s.appendChild(pageAndGslibXsl.importNode(page_xsl.getDocumentElement(), true));
		  root.appendChild(s);
		  
		  Element l = pageAndGslibXsl.createElement("gslibXsl");
		  if (gslib_xsl_doc != null)
		    {
		      Element gslib_xsl_el = gslib_xsl_doc.getDocumentElement();
		      l.appendChild(pageAndGslibXsl.importNode(gslib_xsl_el, true));
		    }
		  root.appendChild(l);

		  if (output.equals("xsl5")) {
		    return converter.getDOM(getStringFromDocument(pageAndGslibXsl));
		  }
		  // actually merge the gslib file with the page
		  XMLTransformer preProcessor = new XMLTransformer();
		  preProcessor.transform_withResultNode(expand_gslib_xsl_doc, pageAndGslibXsl, pageAndGslibDoc);
		}
		if (output.equals("xsl6")) {
		  return converter.getDOM(getStringFromDocument(pageAndGslibDoc));
		}
		  
		pageAndGslibDoc = (Document) transformGSFElements(collection, pageAndGslibDoc, EXPAND_GSF_PASS1_FILE);

		if (output.equals("xsl7")) {
		  return converter.getDOM(getStringFromDocument(pageAndGslibDoc));
		}
		
		pageAndGslibDoc = (Document) transformGSFElements(collection, pageAndGslibDoc, EXPAND_GSF_FILE);
		
		if (output.equals("xsl") || output.equals("skinandlibdocfinal"))
		{
		  return converter.getDOM(getStringFromDocument(pageAndGslibDoc));
		}
		
		if (output.equals("clientside"))
		{
			        
		  // Go through and 'fix up' any 'util:...' or 'java:...' attributes the pageAndGslibDoc has
		  String lang = (String)config_params.get("lang");
		  resolveExtendedNamespaceAttributesXSLT(pageAndGslibDoc,currentInterface,lang); // test= and select= attributes
		  resolveExtendedNamespaceAttributesXML(pageAndGslibDoc,currentInterface,lang);  // href= and src= attributes
		  Node skinAndLibFinal = converter.getDOM(getStringFromDocument(pageAndGslibDoc));
		  
		  // Send XML and skinandlibdoc down the line together
		  Document finalDoc = converter.newDOM();
		  Node finalDocSkin = finalDoc.importNode(pageAndGslibDoc.getDocumentElement(), true);
		  Node finalDocXML = finalDoc.importNode(page_with_xslt_params_doc.getDocumentElement(), true);
		  Element root = finalDoc.createElement("skinlibfinalPlusXML");
		  root.appendChild(finalDocSkin);
		  root.appendChild(finalDocXML);
		  finalDoc.appendChild(root);
		  return (Node) finalDoc.getDocumentElement();
		}
		
		///logger.debug("final xml is ");
		///logger.debug(XMLConverter.getPrettyString(page_xml_doc));

		///logger.debug("final xsl is");
		///logger.debug(XMLConverter.getPrettyString(pageAndGslibDoc));
		
		// The transformer will now work out the resulting doctype from any set in the (merged) stylesheets and
		// will set this in the output document it creates. So don't pass in any docWithDocType to the transformer

  // Here, we finally transform the page xml source with the complete xsl file
  Node finalResult = this.transformer.transform(pageAndGslibDoc, page_xml_doc, config_params);

		if (_debug)
		{
			GSXSLT.fixTables((Document) finalResult);
		}

		return finalResult;

	}

  protected Node generateXSLTClientOutput(Element request) {

    // DocType defaults in case the skin doesn't have an "xsl:output" element
    String qualifiedName = "html";
    String publicID = "-//W3C//DTD HTML 4.01 Transitional//EN";
    String systemID = "http://www.w3.org/TR/html4/loose.dtd";

    // We need to create an empty document with a predefined DocType,
    // that will then be used for the transformation by the DOMResult
    Document docWithDoctype = converter.newDOM(qualifiedName, publicID, systemID);
    String baseURL = request.getAttribute(GSXML.BASE_URL);
				
    // If you're just getting the client-side transform page, why bother with the rest of this?
    Element html = docWithDoctype.createElement("html");
    Element img = docWithDoctype.createElement("img");
    img.setAttribute("src", "loading.gif"); // Make it dynamic
    img.setAttribute("alt", "Please wait...");
    Text title_text = docWithDoctype.createTextNode("Please wait..."); // Make this language dependent
    Element head = docWithDoctype.createElement("head");

    // e.g., <base href="http://localhost:8383/greenstone3/" /><!-- [if lte IE 6]></base><![endif] -->
    Element base = docWithDoctype.createElement("base");
    base.setAttribute("href",baseURL);
    Comment opt_end_base = docWithDoctype.createComment("[if lte IE 6]></base><![endif]");
			
    Element title = docWithDoctype.createElement("title");
    title.appendChild(title_text);

    Element body = docWithDoctype.createElement("body");

    Element jquery_script = docWithDoctype.createElement("script");
    jquery_script.setAttribute("src", "jquery-3.6.0.min.js");
    jquery_script.setAttribute("type", "text/javascript");
    Comment jquery_comment = docWithDoctype.createComment("jQuery");
    jquery_script.appendChild(jquery_comment);

    Element saxonce_script = docWithDoctype.createElement("script");
    saxonce_script.setAttribute("src", "Saxonce/Saxonce.nocache.js");
    saxonce_script.setAttribute("type", "text/javascript");
    Comment saxonce_comment = docWithDoctype.createComment("SaxonCE");
    saxonce_script.appendChild(saxonce_comment);

    Element xsltutil_script = docWithDoctype.createElement("script");
    xsltutil_script.setAttribute("src", "xslt-util.js");
    xsltutil_script.setAttribute("type", "text/javascript");
    Comment xsltutil_comment = docWithDoctype.createComment("JavaScript version of XSLTUtil.java");
    xsltutil_script.appendChild(xsltutil_comment);

    Element script = docWithDoctype.createElement("script");
    Comment script_comment = docWithDoctype.createComment("Filler for browser");
    script.setAttribute("src", "client-side-xslt.js");
    script.setAttribute("type", "text/javascript");
    script.appendChild(script_comment);
			
    Element pagevar = docWithDoctype.createElement("script");
    Element style = docWithDoctype.createElement("style");
    style.setAttribute("type", "text/css");
    Text style_text = docWithDoctype.createTextNode("body { text-align: center; padding: 50px; font: 14pt Arial, sans-serif; font-weight: bold; }");
    pagevar.setAttribute("type", "text/javascript");
    Text page_var_text = docWithDoctype.createTextNode("var placeholder = true;");

    html.appendChild(head);
    head.appendChild(base); head.appendChild(opt_end_base);
    head.appendChild(title);
    head.appendChild(style);
    style.appendChild(style_text);
    html.appendChild(body);
    head.appendChild(pagevar);
    head.appendChild(jquery_script);
    head.appendChild(saxonce_script);
    head.appendChild(xsltutil_script);
    head.appendChild(script);
    pagevar.appendChild(page_var_text);

    body.appendChild(img);
    docWithDoctype.appendChild(html);

    return (Node) docWithDoctype;

  }

  // transform the xsl with xsl to replace all gsf elements. We do this in 2 stages -
  // first do just a text pass, that way we can have gsf elements in the text content 
  protected Node transformGSFElements(String collection, Document source_xsl, String expand_gsf_filename) {

    String expand_gsf_file = GSFile.stylesheetFile(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, expand_gsf_filename);
    Document expand_gsf_doc = this.converter.getDOM(new File(expand_gsf_file));
    
    if (expand_gsf_doc != null)
      {
	return this.transformer.transform(expand_gsf_doc, source_xsl, config_params);
      }
    return source_xsl;
    
  }    

	// method to convert Document to a proper XML string for debug purposes only
	protected String getStringFromDocument(Document doc)
	{
		String content = "";
		try
		{
			DOMSource domSource = new DOMSource(doc);
			StringWriter writer = new StringWriter();
			StreamResult result = new StreamResult(writer);
			TransformerFactory tf = TransformerFactory.newInstance();
			Transformer transformer = tf.newTransformer();
			transformer.transform(domSource, result);
			content = writer.toString();
			System.out.println("Change the & to &Amp; for proper debug display");
			content = StringUtils.replace(content, "&", "&amp;");
			writer.flush();
		}
		catch (TransformerException ex)
		{
			ex.printStackTrace();
			return null;
		}
		return content;
	}

	protected synchronized Document getDoc(String docName) throws Exception
	{
		File xslt_file = new File(docName);

		FileReader reader = new FileReader(xslt_file);
		InputSource xml_source = new InputSource(reader);
		this.parser.parse(xml_source);
		Document doc = this.parser.getDocument();

		return doc;
	}

  protected String getXSLTFilename(String action, String subaction) {
    String name = null;
    if (!subaction.equals(""))
      {
	String key = action + ":" + subaction;
	name = this.xslt_map.get(key);
      }
    // try the action by itself
    if (name == null)
      {
	name = this.xslt_map.get(action);
      }
    if (name == null)
      {
	// so we can reandomly create any named page
	if (action.equals("p") && !subaction.equals(""))
	  {
	    // TODO: pages/ won't work for interface other than default!!
	    name = "pages/" + subaction + ".xsl";
	  }
	
      }
    return name;
  }

  
	protected Document getXSLTDocument(String action, String subaction, String collection)
	{
	  String name = getXSLTFilename(action, subaction);
		Document finalDoc = null;
		if(name != null)
		{
		  // this finds all the stylesheets named "name" and merges them together, in the order of
		  // base interface, current interface, site, collection - the latter overriding the former.
		  // templates with the same name will replace earlier versions
		  finalDoc = GSXSLT.mergedXSLTDocumentCascade(name, (String) this.config_params.get(GSConstants.SITE_NAME), collection, (String) this.config_params.get(GSConstants.INTERFACE_NAME), base_interfaces, _debug);
		}
		return finalDoc;
	}

	// returns the path to the gslib.xsl file that is applicable for the current interface
	protected String getGSLibXSLFilename()
	{
		return GSFile.xmlTransformDir(GSFile.interfaceHome(GlobalProperties.getGSDL3Home(), (String) this.config_params.get(GSConstants.INTERFACE_NAME))) + File.separatorChar + "gslib.xsl";
	}

	// Call this when a FileNotFoundException could be thrown when loading an xsl (xml) file.
	// Returns an error xhtml page indicating which xsl (or other xml) file is missing.
	protected Document fileNotFoundErrorPage(String filenameMessage)
	{
		String errorMessage = "ERROR missing file: " + filenameMessage;
		Element errPage = XMLTransformer.constructErrorXHTMLPage(errorMessage);
		logger.error(errorMessage);
		return errPage.getOwnerDocument();
	}
}